diff options
323 files changed, 4485 insertions, 3798 deletions
diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 486a65e97aa..5b8bd2032d6 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -175,7 +175,6 @@ <include>com.yahoo.vespa:predicate-search-core:*:jar:provided</include> <include>com.yahoo.vespa:processing:*:jar:provided</include> <include>com.yahoo.vespa:provided-dependencies:*:jar:provided</include> - <include>com.yahoo.vespa:provided-yahoo-dependencies:*:jar:provided</include> <include>com.yahoo.vespa:searchcore:*:jar:provided</include> <include>com.yahoo.vespa:searchlib:*:jar:provided</include> <include>com.yahoo.vespa:security-utils:*:jar:provided</include> diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java index 4091363128e..b04f04abfb6 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterController.java @@ -58,7 +58,7 @@ public class ClusterController extends AbstractComponent if (controller == null) { StatusHandler.ContainerStatusPageServer statusPageServer = new StatusHandler.ContainerStatusPageServer(); - controller = FleetController.createForContainer(options, statusPageServer, metricWrapper); + controller = FleetController.create(options, statusPageServer, metricWrapper); controllers.put(clusterName, controller); status.put(clusterName, statusPageServer); } else { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 658dd10f7e5..9c9e1042c79 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -1,10 +1,9 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; import com.yahoo.document.FixedBucketSpaces; import com.yahoo.exception.ExceptionUtils; import com.yahoo.jrt.ListenFailedException; -import java.util.logging.Level; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; @@ -28,7 +27,6 @@ import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageRespon import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServerInterface; import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; -import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter; import java.io.FileNotFoundException; import java.util.ArrayDeque; @@ -44,6 +42,7 @@ import java.util.Set; import java.util.TimeZone; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -51,7 +50,7 @@ import java.util.stream.Stream; public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAddedOrRemovedListener, SystemStateListener, Runnable, RemoteClusterControllerTaskScheduler { - private static Logger log = Logger.getLogger(FleetController.class.getName()); + private static final Logger log = Logger.getLogger(FleetController.class.getName()); private final Timer timer; private final Object monitor; @@ -68,7 +67,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd private final DatabaseHandler database; private final MasterElectionHandler masterElectionHandler; private Thread runner = null; - private AtomicBoolean running = new AtomicBoolean(true); + private final AtomicBoolean running = new AtomicBoolean(true); private FleetControllerOptions options; private FleetControllerOptions nextOptions; private final List<SystemStateListener> systemStateListeners = new CopyOnWriteArrayList<>(); @@ -79,12 +78,12 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd private Long controllerThreadId = null; private boolean waitingForCycle = false; - private StatusPageServer.PatternRequestRouter statusRequestRouter = new StatusPageServer.PatternRequestRouter(); + private final StatusPageServer.PatternRequestRouter statusRequestRouter = new StatusPageServer.PatternRequestRouter(); private final List<ClusterStateBundle> newStates = new ArrayList<>(); private final List<ClusterStateBundle> convergedStates = new ArrayList<>(); private long configGeneration = -1; private long nextConfigGeneration = -1; - private Queue<RemoteClusterControllerTask> remoteTasks = new LinkedList<>(); + private final Queue<RemoteClusterControllerTask> remoteTasks = new LinkedList<>(); private final MetricUpdater metricUpdater; private boolean isMaster = false; @@ -92,9 +91,9 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd private long firstAllowedStateBroadcast = Long.MAX_VALUE; private long tickStartTime = Long.MAX_VALUE; - private List<RemoteClusterControllerTask> tasksPendingStateRecompute = new ArrayList<>(); + private final List<RemoteClusterControllerTask> tasksPendingStateRecompute = new ArrayList<>(); // Invariant: queued task versions are monotonically increasing with queue position - private Queue<VersionDependentTaskCompletion> taskCompletionQueue = new ArrayDeque<>(); + private final Queue<VersionDependentTaskCompletion> taskCompletionQueue = new ArrayDeque<>(); // Legacy behavior is an empty set of explicitly configured bucket spaces, which means that // only a baseline cluster state will be sent from the controller and no per-space state @@ -125,8 +124,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd SystemStateBroadcaster systemStateBroadcaster, MasterElectionHandler masterElectionHandler, MetricUpdater metricUpdater, - FleetControllerOptions options) throws Exception - { + FleetControllerOptions options) { log.info("Starting up cluster controller " + options.fleetControllerIndex + " for cluster " + cluster.getName()); this.timer = timer; this.monitor = timer; @@ -166,26 +164,10 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd propagateOptions(); } - public static FleetController createForContainer(FleetControllerOptions options, - StatusPageServerInterface statusPageServer, - MetricReporter metricReporter) throws Exception { + public static FleetController create(FleetControllerOptions options, + StatusPageServerInterface statusPageServer, + MetricReporter metricReporter) throws Exception { Timer timer = new RealTimer(); - return create(options, timer, statusPageServer, null, metricReporter); - } - - public static FleetController createForStandAlone(FleetControllerOptions options) throws Exception { - Timer timer = new RealTimer(); - RpcServer rpcServer = new RpcServer(timer, timer, options.clusterName, options.fleetControllerIndex, options.slobrokBackOffPolicy); - StatusPageServer statusPageServer = new StatusPageServer(timer, timer, options.httpPort); - return create(options, timer, statusPageServer, rpcServer, new NoMetricReporter()); - } - - private static FleetController create(FleetControllerOptions options, - Timer timer, - StatusPageServerInterface statusPageServer, - RpcServer rpcServer, - MetricReporter metricReporter) throws Exception - { MetricUpdater metricUpdater = new MetricUpdater(metricReporter, options.fleetControllerIndex); EventLog log = new EventLog(timer, metricUpdater); ContentCluster cluster = new ContentCluster( @@ -209,7 +191,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd SystemStateBroadcaster stateBroadcaster = new SystemStateBroadcaster(timer, timer); MasterElectionHandler masterElectionHandler = new MasterElectionHandler(options.fleetControllerIndex, options.fleetControllerCount, timer, timer); FleetController controller = new FleetController( - timer, log, cluster, stateGatherer, communicator, statusPageServer, rpcServer, lookUp, database, stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); + timer, log, cluster, stateGatherer, communicator, statusPageServer, null, lookUp, database, stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); controller.start(); return controller; } @@ -469,7 +451,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } /** This is called when the options field has been set to a new set of options */ - private void propagateOptions() throws java.io.IOException, ListenFailedException { + private void propagateOptions() { verifyInControllerThread(); if (changesConfiguredNodeSet(options.nodes)) { @@ -547,7 +529,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } catch (Exception e) { responseCode = StatusPageResponse.ResponseCode.INTERNAL_SERVER_ERROR; message = "Internal Server Error"; - hiddenMessage = ExceptionUtils.getStackTraceAsString(e);; + hiddenMessage = ExceptionUtils.getStackTraceAsString(e); log.log(Level.FINE, "Unknown exception thrown for request " + httpRequest.getRequest() + ": " + hiddenMessage); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java index 553b3332ee8..2044eb1eab0 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java @@ -1,16 +1,20 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizone Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.slobrok.api.BackOffPolicy; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.NodeType; -import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; -import java.time.Duration; -import java.util.*; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; +import java.time.Duration; +import java.util.Collection; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; /** * This class represents all the options that can be set in the fleetcontroller. @@ -30,7 +34,7 @@ public class FleetControllerOptions implements Cloneable { public int stateGatherCount = 2; // TODO: This cannot be null but nonnull is not verified - public String slobrokConnectionSpecs[]; + public String[] slobrokConnectionSpecs; public int rpcPort = 0; public int httpPort = 0; public int distributionBits = 16; @@ -189,7 +193,7 @@ public class FleetControllerOptions implements Cloneable { static DecimalFormat DecimalDot2 = new DecimalFormat("0.00", new DecimalFormatSymbols(Locale.ENGLISH)); - public void writeHtmlState(StringBuilder sb, StatusPageServer.HttpRequest request) { + public void writeHtmlState(StringBuilder sb) { String slobrokspecs = ""; for (int i=0; i<slobrokConnectionSpecs.length; ++i) { if (i != 0) slobrokspecs += "<br>"; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java index f799492d164..9d5e5e46f08 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java @@ -22,7 +22,7 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa private final EventLog eventLog; private final long startedTime; private final RunDataExtractor data; - private boolean showLocalSystemStatesInLog = true; + private final boolean showLocalSystemStatesInLog; public LegacyIndexPageRequestHandler(Timer timer, boolean showLocalSystemStatesInLog, ContentCluster cluster, MasterElectionHandler masterElectionHandler, @@ -72,12 +72,12 @@ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHa eventLog ); // Overview of current config - data.getOptions().writeHtmlState(content, request); + data.getOptions().writeHtmlState(content); // Current cluster state and cluster state history writeHtmlState(stateVersionTracker, content, request); } else { // Overview of current config - data.getOptions().writeHtmlState(content, request); + data.getOptions().writeHtmlState(content); } // Event log eventLog.writeHtmlState(content, null); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index 3d3de32c356..5920a7d651a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -1,25 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.status.statuspage; -import com.yahoo.exception.ExceptionUtils; -import java.util.logging.Level; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.SocketTimeoutException; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TimeZone; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -27,238 +12,10 @@ import java.util.regex.Pattern; /** * Shows status pages with debug information through a very simple HTTP interface. */ -public class StatusPageServer implements Runnable, StatusPageServerInterface { +public class StatusPageServer { public static Logger log = Logger.getLogger(StatusPageServer.class.getName()); - private final com.yahoo.vespa.clustercontroller.core.Timer timer; - private final Object monitor; - private ServerSocket ssocket; - private final Thread runner; - private int port = 0; - private boolean running = true; - private boolean shouldBeConnected = false; - private HttpRequest currentHttpRequest = null; - private StatusPageResponse currentResponse = null; - private long lastConnectErrorTime = 0; - private String lastConnectError = ""; - private PatternRequestRouter staticContentRouter = new PatternRequestRouter(); - private Date startTime = new Date(); - - public StatusPageServer(com.yahoo.vespa.clustercontroller.core.Timer timer, Object monitor, int port) throws java.io.IOException, InterruptedException { - this.timer = timer; - this.monitor = monitor; - this.port = port; - connect(); - runner = new Thread(this); - runner.start(); - } - - public boolean isConnected() { - if (ssocket != null && ssocket.isBound() && (ssocket.getLocalPort() == port || port == 0)) { - return true; - } else { - log.log(Level.FINEST, "Status page server socket is no longer connected: "+ (ssocket != null) + " " + ssocket.isBound() + " " + ssocket.getLocalPort() + " " + port); - return false; - } - } - - public void connect() throws java.io.IOException, InterruptedException { - synchronized(monitor) { - if (ssocket != null) { - if (ssocket.isBound() && ssocket.getLocalPort() == port) { - return; - } - disconnect(); - } - ssocket = new ServerSocket(); - if (port != 0) { - ssocket.setReuseAddress(true); - } - ssocket.setSoTimeout(100); - ssocket.bind(new InetSocketAddress(port)); - shouldBeConnected = true; - for (int i=0; i<200; ++i) { - if (isConnected()) break; - Thread.sleep(10); - } - if (!isConnected()) { - log.log(Level.INFO, "Fleetcontroller: Server Socket not ready after connect()"); - } - log.log(Level.FINE, "Fleet controller status page viewer listening to " + ssocket.getLocalSocketAddress()); - monitor.notifyAll(); - } - } - - public void disconnect() throws java.io.IOException { - synchronized(monitor) { - shouldBeConnected = false; - if (ssocket != null) ssocket.close(); - ssocket = null; - monitor.notifyAll(); - } - } - - public void setPort(int port) throws java.io.IOException, InterruptedException { - // Only bother to reconnect if we were connected to begin with, we care about what port it runs on, and it's not already running there - if (port != 0 && isConnected() && port != ((InetSocketAddress) ssocket.getLocalSocketAddress()).getPort()) { - log.log(Level.INFO, "Exchanging port used by status server. Moving from port " - + ((InetSocketAddress) ssocket.getLocalSocketAddress()).getPort() + " to port " + port); - disconnect(); - this.port = port; - if (ssocket == null || !ssocket.isBound() || ssocket.getLocalPort() != port) { - connect(); - } - } else { - this.port = port; - } - } - - public int getPort() { - // Cannot use this.port, because of tests using port 0 to get any address - if (ssocket == null || !ssocket.isBound()) { - throw new IllegalStateException("Cannot ask for port before server socket is bound"); - } - return ((InetSocketAddress) ssocket.getLocalSocketAddress()).getPort(); - } - - public void shutdown() throws InterruptedException, java.io.IOException { - running = false; - runner.interrupt(); - runner.join(); - disconnect(); - } - - public void run() { - try{ - while (running) { - Socket connection = null; - ServerSocket serverSocket = null; - synchronized(monitor) { - if (ssocket == null || !ssocket.isBound()) { - monitor.wait(1000); - continue; - } - serverSocket = ssocket; - } - try{ - connection = serverSocket.accept(); - } catch (SocketTimeoutException e) { - // Ignore, since timeout is set to 100 ms - } catch (java.io.IOException e) { - log.log(shouldBeConnected ? Level.WARNING : Level.FINE, "Caught IO exception in ServerSocket.accept(): " + e.getMessage()); - } - if (connection == null) continue; - log.log(Level.FINE, "Got a status page request."); - String requestString = ""; - OutputStream output = null; - try (BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream()))) { - StringBuilder sb = new StringBuilder(); - while (true) { - String s = br.readLine(); - if (s == null) throw new java.io.IOException("No data in HTTP request on socket " + connection.toString()); - if (s.length() > 4 && s.substring(0,4).equals("GET ")) { - int nextSpace = s.indexOf(' ', 4); - if (nextSpace == -1) { - requestString = s.substring(4); - } else { - requestString = s.substring(4, nextSpace); - } - } - if (s == null || s.equals("")) break; - sb.append(s).append("\n"); - } - log.log(Level.FINE, "Got HTTP request: " + sb.toString()); - - HttpRequest httpRequest = null; - StatusPageResponse response = null; - try { - httpRequest = new HttpRequest(requestString); - // Static files are served directly by the HTTP server thread, since - // it makes no sense to go via the fleetcontroller logic for these. - RequestHandler contentHandler = staticContentRouter.resolveHandler(httpRequest); - if (contentHandler != null) { - response = contentHandler.handle(httpRequest); - } - } catch (Exception e) { - response = new StatusPageResponse(); - response.setResponseCode(StatusPageResponse.ResponseCode.INTERNAL_SERVER_ERROR); - StringBuilder content = new StringBuilder(); - response.writeHtmlHeader(content, "Internal Server Error"); - response.writeHtmlFooter(content, ExceptionUtils.getStackTraceAsString(e)); - response.writeContent(content.toString()); - } - if (response == null) { - synchronized(monitor) { - currentHttpRequest = httpRequest; - currentResponse = null; - while (running) { - if (currentResponse != null) { - response = currentResponse; - break; - } - monitor.wait(100); - } - } - } - if (response == null) { - response = new StatusPageResponse(); - StringBuilder content = new StringBuilder(); - response.setContentType("text/html"); - response.writeHtmlHeader(content, "Failed to get response. Fleet controller probably in the process of shutting down."); - response.writeHtmlFooter(content, ""); - response.writeContent(content.toString()); - } - - output = connection.getOutputStream(); - StringBuilder header = new StringBuilder(); - // TODO: per-response cache control - header.append("HTTP/1.1 ") - .append(response.getResponseCode().getCode()) - .append(" ") - .append(response.getResponseCode().getMessage()) - .append("\r\n") - .append("Date: ").append(new Date().toString()).append("\r\n") - .append("Connection: Close\r\n") - .append("Content-type: ").append(response.getContentType()).append("\r\n"); - if (response.isClientCachingEnabled()) { - // TODO(vekterli): would be better to let HTTP handlers set header values in response - DateFormat df = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z"); - df.setTimeZone(TimeZone.getTimeZone("GMT")); - header.append("Last-Modified: ").append(df.format(startTime)).append("\r\n"); - } else { - header.append("Expires: Fri, 01 Jan 1990 00:00:00 GMT\r\n") - .append("Pragma: no-cache\r\n") - .append("Cache-control: no-cache, must-revalidate\r\n"); - } - header.append("\r\n"); - output.write(header.toString().getBytes()); - output.write(response.getOutputStream().toByteArray()); - } catch (java.io.IOException e) { - log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? Level.FINE : Level.INFO, - "Failed to process HTTP request : " + e.getMessage()); - } catch (Exception e) { - log.log(Level.WARNING, "Caught exception in HTTP server thread: " - + e.getClass().getName() + ": " + e.getMessage()); - } finally { - if (output != null) try { - output.close(); - } catch (IOException e) { - log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? Level.FINE : Level.INFO, - "Failed to close output stream on socket " + connection + ": " + e.getMessage()); - } - if (connection != null) try{ - connection.close(); - } catch (IOException e) { - log.log(Level.INFO, "Failed to close socket " + connection + ": " + e.getMessage()); - } - } - } - } catch (InterruptedException e) { - log.log(Level.FINE, "Status processing thread shut down by interrupt exception: " + e); - } - } - /** * Very simple HTTP request class. This should be replaced the second * the fleetcontroller e.g. moves into the container. @@ -377,29 +134,4 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { } } - public HttpRequest getCurrentHttpRequest() { - synchronized (monitor) { - return currentHttpRequest; - } - } - - public void answerCurrentStatusRequest(StatusPageResponse r) { - if (!isConnected()) { - long time = timer.getCurrentTimeInMillis(); - try{ - connect(); - } catch (Exception e) { - if (!e.getMessage().equals(lastConnectError) || time - lastConnectErrorTime > 60 * 1000) { - lastConnectError = e.getMessage(); - lastConnectErrorTime = time; - log.log(Level.WARNING, "Failed to initialize HTTP status server server socket: " + e.getMessage()); - } - } - } - synchronized (monitor) { - currentResponse = r; - currentHttpRequest = null; // Avoid fleetcontroller processing request more than once - } - } - } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index 1587e2696b8..f8bf387ce41 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -9,7 +9,6 @@ import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.BackOffPolicy; import com.yahoo.jrt.slobrok.server.Slobrok; -import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.state.ClusterState; @@ -22,7 +21,7 @@ import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator; import com.yahoo.vespa.clustercontroller.core.rpc.RpcServer; import com.yahoo.vespa.clustercontroller.core.rpc.SlobrokClient; -import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; +import com.yahoo.vespa.clustercontroller.core.status.StatusHandler; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServerInterface; import com.yahoo.vespa.clustercontroller.core.testutils.WaitCondition; import com.yahoo.vespa.clustercontroller.core.testutils.WaitTask; @@ -40,9 +39,11 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -149,6 +150,7 @@ public abstract class FleetControllerTest implements Waiter { } FleetController createFleetController(boolean useFakeTimer, FleetControllerOptions options, boolean startThread, StatusPageServerInterface status) throws Exception { + Objects.requireNonNull(status, "status server cannot be null"); Timer timer = useFakeTimer ? this.timer : new RealTimer(); MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex); EventLog log = new EventLog(timer, metricUpdater); @@ -169,9 +171,6 @@ public abstract class FleetControllerTest implements Waiter { options.nodeStateRequestRoundTripTimeMaxSeconds); SlobrokClient lookUp = new SlobrokClient(timer); lookUp.setSlobrokConnectionSpecs(new String[0]); - if (status == null) { - status = new StatusPageServer(timer, timer, options.httpPort); - } RpcServer rpcServer = new RpcServer(timer, timer, options.clusterName, options.fleetControllerIndex, options.slobrokBackOffPolicy); DatabaseHandler database = new DatabaseHandler(new ZooKeeperDatabaseFactory(), timer, options.zooKeeperServerAddress, options.fleetControllerIndex, timer); StateChangeHandler stateGenerator = new StateChangeHandler(timer, log, metricUpdater); @@ -189,7 +188,7 @@ public abstract class FleetControllerTest implements Waiter { } protected void setUpFleetController(boolean useFakeTimer, FleetControllerOptions options, boolean startThread) throws Exception { - setUpFleetController(useFakeTimer, options, startThread, null); + setUpFleetController(useFakeTimer, options, startThread, new StatusHandler.ContainerStatusPageServer()); } protected void setUpFleetController(boolean useFakeTimer, FleetControllerOptions options, boolean startThread, StatusPageServerInterface status) throws Exception { if (slobrok == null) setUpSystem(useFakeTimer, options); @@ -209,7 +208,7 @@ public abstract class FleetControllerTest implements Waiter { void startFleetController() throws Exception { if (fleetController == null) { - fleetController = createFleetController(usingFakeTimer, options, true, null); + fleetController = createFleetController(usingFakeTimer, options, true, new StatusHandler.ContainerStatusPageServer()); } else { log.log(Level.WARNING, "already started fleetcontroller, not starting another"); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index d14f6701288..896f73ce6bf 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -12,6 +12,7 @@ import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; +import com.yahoo.vespa.clustercontroller.core.status.StatusHandler; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -60,7 +61,7 @@ public class MasterElectionTest extends FleetControllerTest { for (int i=0; i<count; ++i) { FleetControllerOptions nodeOptions = options.clone(); nodeOptions.fleetControllerIndex = i; - fleetControllers.add(createFleetController(usingFakeTimer, nodeOptions, true, null)); + fleetControllers.add(createFleetController(usingFakeTimer, nodeOptions, true, new StatusHandler.ContainerStatusPageServer())); } } @@ -143,14 +144,15 @@ public class MasterElectionTest extends FleetControllerTest { assertFalse("Fleet controller " + i, fleetControllers.get(i).isMaster()); } + StatusHandler.ContainerStatusPageServer statusPageServer = new StatusHandler.ContainerStatusPageServer(); log.log(Level.INFO, "STARTING FLEET CONTROLLER 2"); - fleetControllers.set(2, createFleetController(usingFakeTimer, fleetControllers.get(2).getOptions(), true, null)); + fleetControllers.set(2, createFleetController(usingFakeTimer, fleetControllers.get(2).getOptions(), true, statusPageServer)); waitForMaster(2); log.log(Level.INFO, "STARTING FLEET CONTROLLER 0"); - fleetControllers.set(0, createFleetController(usingFakeTimer, fleetControllers.get(0).getOptions(), true, null)); + fleetControllers.set(0, createFleetController(usingFakeTimer, fleetControllers.get(0).getOptions(), true, statusPageServer)); waitForMaster(0); log.log(Level.INFO, "STARTING FLEET CONTROLLER 1"); - fleetControllers.set(1, createFleetController(usingFakeTimer, fleetControllers.get(1).getOptions(), true, null)); + fleetControllers.set(1, createFleetController(usingFakeTimer, fleetControllers.get(1).getOptions(), true, statusPageServer)); waitForMaster(0); log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 4"); @@ -538,7 +540,7 @@ public class MasterElectionTest extends FleetControllerTest { waitForMaster(1); waitForCompleteCycle(1); - fleetControllers.set(0, createFleetController(usingFakeTimer, fleetControllers.get(0).getOptions(), true, null)); + fleetControllers.set(0, createFleetController(usingFakeTimer, fleetControllers.get(0).getOptions(), true, new StatusHandler.ContainerStatusPageServer())); waitForMaster(0); waitForCompleteCycle(0); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StatusPagesTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StatusPagesTest.java deleted file mode 100644 index f761538cf1e..00000000000 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StatusPagesTest.java +++ /dev/null @@ -1,374 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.clustercontroller.core; - -import com.yahoo.vdslib.distribution.Distribution; -import com.yahoo.vdslib.state.Node; -import com.yahoo.vdslib.state.NodeState; -import com.yahoo.vdslib.state.NodeType; -import com.yahoo.vdslib.state.State; -import com.yahoo.vespa.clustercontroller.core.status.StatusHandler; -import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageResponse; -import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; -import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest; -import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpResult; -import org.codehaus.jettison.json.JSONObject; -import org.junit.Test; - -import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.net.Socket; -import java.nio.charset.StandardCharsets; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.TimeZone; -import java.util.logging.Logger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -public class StatusPagesTest extends FleetControllerTest { - - public static Logger log = Logger.getLogger(StatusPagesTest.class.getName()); - - private String doHttpGetRequest(String request, Date ifModifiedSince) throws IOException { - int statusPort = fleetController.getHttpPort(); - Socket socket = new Socket("localhost", statusPort); - - BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream())); - bw.write("GET " + request + " HTTP/1.1\r\n"); - if (ifModifiedSince != null) { - DateFormat df = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z"); - df.setTimeZone(TimeZone.getTimeZone("GMT")); - bw.write("If-Modified-Since: " + df.format(ifModifiedSince) + "\r\n"); - } - bw.write("\r\n"); - bw.flush(); - - InputStream stream = socket.getInputStream(); - ByteArrayOutputStream output = new ByteArrayOutputStream(); - try { - byte [] buf = new byte[4096]; - while (true) { - int read = stream.read(buf); - if (read<=0) { - break; - } - output.write(buf, 0, read); - } - output.close(); - return output.toString(); - } finally { - stream.close(); - bw.close(); - } - } - - private String doHttpGetRequest(String request) throws IOException { - return doHttpGetRequest(request, null); - } - - @Test - public void testStatusThroughContainer() throws Exception { - startingTest("StatusPagesTest::testStatusThroughContainer()"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.setStorageDistribution(new Distribution(Distribution.getDefaultDistributionConfig(3, 10))); - final StatusHandler.ContainerStatusPageServer statusServer = new StatusHandler.ContainerStatusPageServer(); - setUpFleetController(true, options, true, statusServer); - setUpVdsNodes(true, new DummyVdsNodeOptions()); - waitForStableSystem(); - - //ThreadPoolExecutor executor = new ThreadPoolExecutor(10, 100, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(1000)); - //FleetControllerComponent fcComp = new FleetControllerComponent(); - //fcComp.addFleetController("mycluster", fleetController, statusServer); - StatusHandler comp = new StatusHandler(new StatusHandler.ClusterStatusPageServerSet() { - @Override - public StatusHandler.ContainerStatusPageServer get(String cluster) { - return ("mycluster".equals(cluster) ? statusServer : null); - } - - @Override - public Map<String, StatusHandler.ContainerStatusPageServer> getAll() { - Map<String, StatusHandler.ContainerStatusPageServer> map = new HashMap<>(); - map.put("mycluster", statusServer); - return map; - } - }); - - { - HttpRequest request = new HttpRequest().setPath("/clustercontroller-status/v1"); - HttpResult result = comp.handleRequest(request); - assertEquals(result.toString(true), 200, result.getHttpReturnCode()); - assertEquals("<title>clusters</title>\n<a href=\"./mycluster\">mycluster</a><br>\n", result.getContent().toString()); - } - { - HttpRequest request = new HttpRequest().setPath("/clustercontroller-status/v1/"); - HttpResult result = comp.handleRequest(request); - assertEquals(result.toString(true), 200, result.getHttpReturnCode()); - assertEquals("<title>clusters</title>\n<a href=\"./mycluster\">mycluster</a><br>\n", result.getContent().toString()); - } - { - HttpRequest request = new HttpRequest().setPath("/clustercontroller-status/v1/mycluster"); - HttpResult result = comp.handleRequest(request); - assertEquals(result.toString(true), 200, result.getHttpReturnCode()); - assertTrue(result.toString(true), result.getContent().toString().contains( - "mycluster Cluster Controller 0 Status Page")); - } - { - HttpRequest request = new HttpRequest().setPath("/clustercontroller-status/v1/mycluster/"); - HttpResult result = comp.handleRequest(request); - assertEquals(result.toString(true), 200, result.getHttpReturnCode()); - assertTrue(result.toString(true), result.getContent().toString().contains( - "mycluster Cluster Controller 0 Status Page")); - assertTrue(result.toString(true), result.getContent().toString().contains( - "href=\"mycluster/node=distributor.0\"")); - assertTrue(result.toString(true), result.getContent().toString().contains( - "href=\"mycluster/node=storage.0\"")); - } - { - HttpRequest request = new HttpRequest().setPath("/clustercontroller-status/v1/mycluster/node=storage.0"); - HttpResult result = comp.handleRequest(request); - assertEquals(result.toString(true), 200, result.getHttpReturnCode()); - assertTrue(result.toString(true), result.getContent().toString().contains( - "Node status for storage.0")); - assertTrue(result.toString(true), result.getContent().toString().contains( - "href=\"..\"")); - } - { - HttpRequest request = new HttpRequest().setPath("/clustercontroller-status/v1/foo"); - HttpResult result = comp.handleRequest(request); - assertEquals(result.toString(true), 404, result.getHttpReturnCode()); - } - { - HttpRequest request = new HttpRequest().setPath("/foobar/v1/mycluster/"); - HttpResult result = comp.handleRequest(request); - assertEquals(result.toString(true), 404, result.getHttpReturnCode()); - } - { - HttpRequest request = new HttpRequest().setPath("/clustercontroller-status/v2/"); - HttpResult result = comp.handleRequest(request); - assertEquals(result.toString(true), 404, result.getHttpReturnCode()); - } - //executor.shutdown(); - } - - @Test - public void testZooKeeperAddressSplitting() { - String rawAddress = "conc1.foo.yahoo.com:2181,conc2.foo.yahoo.com:2181," - + "dp1.foo.yahoo.com:2181,dp2.foo.yahoo.com:2181," - + "dp3.foo.yahoo.com:2181"; - String result = "conc1.foo.yahoo.com:2181, conc2.foo.yahoo.com:2181, " - + "dp1.foo.yahoo.com:2181, dp2.foo.yahoo.com:2181, " - + "dp3.foo.yahoo.com:2181"; - String split = FleetControllerOptions.splitZooKeeperAddress(rawAddress); - assertEquals(result, split); - } - - @Test - public void testSimpleConnectionWithSomeContent() throws Exception { - // Set this to true temporary if you want to check status page from browser. Should be false in checked in code always. - boolean haltTestToViewStatusPage = false; - startingTest("StatusPagesTest::testSimpleConnectionWithSomeContent()"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.setStorageDistribution(new Distribution(Distribution.getDefaultDistributionConfig(3, 10))); - //options.minRatioOfStorageNodesUp = 0.99; - if (haltTestToViewStatusPage) { - options.httpPort = 19234; - } - setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); - waitForStableSystem(); - - nodes.get(2).disconnectBreakConnection(); - nodes.get(5).disconnectAsShutdown(); - nodes.get(7).disconnectSlobrok(); - - fleetController.getCluster().getNodeInfo(new Node(NodeType.STORAGE, 3)).setWantedState(new NodeState(NodeType.STORAGE, State.MAINTENANCE).setDescription("Test&<>special")); - - String content = doHttpGetRequest("/"); - - assertTrue(content, content.contains("<html>")); - assertTrue(content, content.contains("</html>")); - assertTrue(content, content.contains("Baseline cluster state")); - assertTrue(content, content.contains("Cluster states")); - assertTrue(content, content.contains("Event log")); - - if (haltTestToViewStatusPage) { - System.err.println(content); - try{ - Thread.sleep(1000000); - } catch (InterruptedException e) {} - } - } - - @Test - public void testNodePage() throws Exception { - startingTest("StatusPagesTest::testNodePage()"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.setStorageDistribution(new Distribution(Distribution.getDefaultDistributionConfig(3, 10))); - setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); - waitForStableSystem(); - - String content = doHttpGetRequest("/node=storage.0"); - - assertTrue(content, content.contains("<html>")); - assertTrue(content, content.contains("</html>")); - assertTrue(content, content.contains("Node status for storage.0")); - assertTrue(content, content.contains("REPORTED")); - assertTrue(content, content.contains("Altered node state in cluster state from")); - //System.err.println(sb.toString()); - } - - @Test - public void testErrorResponseCode() throws Exception { - startingTest("StatusPagesTest::testNodePage()"); - FleetControllerOptions options = defaultOptions("mycluster"); - options.setStorageDistribution(new Distribution(Distribution.getDefaultDistributionConfig(3, 10))); - setUpFleetController(true, options); - setUpVdsNodes(true, new DummyVdsNodeOptions()); - waitForStableSystem(); - - String content = doHttpGetRequest("/fraggle/rock"); - - assertTrue(content.contains("404 Not Found")); - //System.err.println(sb.toString()); - } - - private StatusPageServer.HttpRequest makeHttpRequest(String request) { - return new StatusPageServer.HttpRequest(request); - } - - @Test - public void testHttpRequestParsing() { - { - StatusPageServer.HttpRequest request = makeHttpRequest("/") ; - assertEquals("/", request.getPath()); - assertFalse(request.hasQueryParameters()); - } - { - StatusPageServer.HttpRequest request = makeHttpRequest("/foo/bar"); - assertEquals("/foo/bar", request.getPath()); - assertFalse(request.hasQueryParameters()); - } - { - StatusPageServer.HttpRequest request = makeHttpRequest("/foo/bar?baz=baff"); - assertEquals("/foo/bar", request.getPath()); - assertTrue(request.hasQueryParameters()); - assertEquals("baff", request.getQueryParameter("baz")); - } - { - StatusPageServer.HttpRequest request = makeHttpRequest("/?baz=baff&blarg=blee"); - assertEquals("/", request.getPath()); - assertTrue(request.hasQueryParameters()); - assertEquals("baff", request.getQueryParameter("baz")); - assertEquals("blee", request.getQueryParameter("blarg")); - } - { - StatusPageServer.HttpRequest request = makeHttpRequest("/node=storage.101?showlocal"); - assertEquals("/node=storage.101", request.getPath()); - assertTrue(request.hasQueryParameters()); - assertTrue(request.hasQueryParameter("showlocal")); - assertNull(request.getQueryParameter("showlocal")); - } - } - - private static class DummyRequestHandler implements StatusPageServer.RequestHandler { - private String returnData; - DummyRequestHandler(String returnData) { - this.returnData = returnData; - } - - @Override - public StatusPageResponse handle(StatusPageServer.HttpRequest request) { - StatusPageResponse response = new StatusPageResponse(); - response.writeContent(returnData); - return response; - } - } - - private String invokeHandler(StatusPageServer.RequestRouter router, String request) { - StatusPageServer.HttpRequest httpRequest = makeHttpRequest(request); - StatusPageServer.RequestHandler handler = router.resolveHandler(httpRequest); - if (handler == null) { - return null; - } - return handler.handle(httpRequest).getOutputStream().toString(StandardCharsets.UTF_8); - } - - @Test - public void testRequestRouting() { - StatusPageServer.PatternRequestRouter router = new StatusPageServer.PatternRequestRouter(); - router.addHandler("^/alerts/red.*", new DummyRequestHandler("red alert!")); - router.addHandler("^/alerts.*", new DummyRequestHandler("beige alert")); - router.addHandler("^/$", new DummyRequestHandler("root")); - assertEquals("root", invokeHandler(router, "/")); - assertEquals("beige alert", invokeHandler(router, "/alerts")); - assertEquals("beige alert", invokeHandler(router, "/alerts?foo")); - assertEquals("red alert!", invokeHandler(router, "/alerts/red")); - assertEquals("red alert!", invokeHandler(router, "/alerts/red/blue")); - assertNull(invokeHandler(router, "/blarg")); - } - - private String[] getResponseParts(String response) { - int offset = response.indexOf("\r\n\r\n"); - if (offset == -1) { - throw new IllegalStateException("No HTTP header delimiter found"); - } - return new String[] { - response.substring(0, offset + 2), // all header lines must have linebreaks - response.substring(offset + 4) - }; - } - - @Test - public void testStateServing() throws Exception { - startingTest("StatusPagesTest::testStateServing()"); - FleetControllerOptions options = defaultOptions("mycluster"); - setUpFleetController(true, options); - fleetController.updateOptions(options, 5); - waitForCompleteCycle(); - { - String content = doHttpGetRequest("/state/v1/health"); - String[] parts = getResponseParts(content); - String body = parts[1]; - String expected = - "{\n" + - " \"status\" : {\n" + - " \"code\" : \"up\"\n" + - " },\n" + - " \"config\" : {\n" + - " \"component\" : {\n" + - " \"generation\" : 5\n" + - " }\n" + - " }\n" + - "}"; - assertEquals(expected, body); - // Check that it actually parses - new JSONObject(expected); - } - } - - @Test - public void testClusterStateServing() throws Exception { - startingTest("StatusPagesTest::testClusterStateServing()"); - FleetControllerOptions options = defaultOptions("mycluster"); - setUpFleetController(true, options); - fleetController.updateOptions(options, 5); - waitForCompleteCycle(); - { - String content = doHttpGetRequest("/clusterstate"); - String[] parts = getResponseParts(content); - String body = parts[1]; - String expected = "version:2 cluster:d"; - assertEquals(expected, body); - } - } -} diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java index b2f1d833df7..13ed9800db3 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java @@ -11,6 +11,7 @@ import com.yahoo.documentapi.ProgressToken; import com.yahoo.documentapi.VisitorControlHandler; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.curator.Lock; import java.time.Clock; @@ -44,15 +45,13 @@ public class Reindexer { private final Map<DocumentType, Instant> ready; private final ReindexingCurator database; private final Function<VisitorParameters, Runnable> visitorSessions; + private final ReindexingMetrics metrics; private final Clock clock; private final Phaser phaser = new Phaser(2); // Reindexer and visitor. - private Reindexing reindexing; - private Status status; - @Inject public Reindexer(Cluster cluster, Map<DocumentType, Instant> ready, ReindexingCurator database, - DocumentAccess access, Clock clock) { + DocumentAccess access, Metric metric, Clock clock) { this(cluster, ready, database, @@ -64,11 +63,12 @@ public class Reindexer { throw new IllegalStateException(e); } }, + metric, clock); } Reindexer(Cluster cluster, Map<DocumentType, Instant> ready, ReindexingCurator database, - Function<VisitorParameters, Runnable> visitorSessions, Clock clock) { + Function<VisitorParameters, Runnable> visitorSessions, Metric metric, Clock clock) { for (DocumentType type : ready.keySet()) cluster.bucketSpaceOf(type); // Verifies this is known. @@ -76,10 +76,11 @@ public class Reindexer { this.ready = new TreeMap<>(ready); // Iterate through document types in consistent order. this.database = database; this.visitorSessions = visitorSessions; + this.metrics = new ReindexingMetrics(metric, cluster.name); this.clock = clock; } - /** Lets the reindexere abort any ongoing visit session, wait for it to complete normally, then exit. */ + /** Lets the reindexer abort any ongoing visit session, wait for it to complete normally, then exit. */ public void shutdown() { phaser.forceTermination(); // All parties waiting on this phaser are immediately allowed to proceed. } @@ -90,12 +91,16 @@ public class Reindexer { throw new IllegalStateException("Already shut down"); try (Lock lock = database.lockReindexing()) { + Reindexing reindexing = updateWithReady(ready, database.readReindexing(), clock.instant()); + database.writeReindexing(reindexing); + metrics.dump(reindexing); + for (DocumentType type : ready.keySet()) { // We consider only document types for which we have config. if (ready.get(type).isAfter(clock.instant())) log.log(INFO, "Received config for reindexing which is ready in the future — will process later " + "(" + ready.get(type) + " is after " + clock.instant() + ")"); else - progress(type); + progress(type, new AtomicReference<>(reindexing), new AtomicReference<>(reindexing.status().get(type))); if (phaser.isTerminated()) break; @@ -103,77 +108,86 @@ public class Reindexer { } } + static Reindexing updateWithReady(Map<DocumentType, Instant> ready, Reindexing reindexing, Instant now) { + for (DocumentType type : ready.keySet()) { // We consider update for document types for which we have config. + if ( ! ready.get(type).isAfter(now)) { + Status status = reindexing.status().getOrDefault(type, Status.ready(now) + .running() + .successful(now)); + if (status.startedAt().isBefore(ready.get(type))) + status = Status.ready(now); + + reindexing = reindexing.with(type, status); + } + } + return reindexing; + } + @SuppressWarnings("fallthrough") // (ノಠ∩ಠ)ノ彡( \o°o)\ - private void progress(DocumentType type) { - // If this is a new document type (or a new cluster), no reindexing is required. - reindexing = database.readReindexing(); - status = reindexing.status().getOrDefault(type, - Status.ready(clock.instant()) - .running() - .successful(clock.instant())); - if (ready.get(type).isAfter(status.startedAt())) - status = Status.ready(clock.instant()); // Need to restart, as a newer reindexing is required. - - database.writeReindexing(reindexing = reindexing.with(type, status)); - - switch (status.state()) { + private void progress(DocumentType type, AtomicReference<Reindexing> reindexing, AtomicReference<Status> status) { + + database.writeReindexing(reindexing.updateAndGet(value -> value.with(type, status.get()))); + metrics.dump(reindexing.get()); + + switch (status.get().state()) { default: - log.log(WARNING, "Unknown reindexing state '" + status.state() + "'"); + log.log(WARNING, "Unknown reindexing state '" + status.get().state() + "'"); case FAILED: log.log(FINE, () -> "Not continuing reindexing of " + type + " due to previous failure"); case SUCCESSFUL: // Intentional fallthrough — all three are done states. return; case RUNNING: log.log(WARNING, "Unexpected state 'RUNNING' of reindexing of " + type); - case READY: // Intentional fallthrough — must just assume we failed updating state when exiting previously. + case READY: // Intentional fallthrough — must just assume we failed updating state when exiting previously. log.log(FINE, () -> "Running reindexing of " + type); } // Visit buckets until they're all done, or until we are interrupted. - status = status.running(); + status.updateAndGet(Status::running); AtomicReference<Instant> progressLastStored = new AtomicReference<>(clock.instant()); VisitorControlHandler control = new VisitorControlHandler() { @Override public void onProgress(ProgressToken token) { super.onProgress(token); - status = status.progressed(token); + status.updateAndGet(value -> value.progressed(token)); if (progressLastStored.get().isBefore(clock.instant().minusSeconds(10))) { progressLastStored.set(clock.instant()); - database.writeReindexing(reindexing = reindexing.with(type, status)); + database.writeReindexing(reindexing.updateAndGet(value -> value.with(type, status.get()))); + metrics.dump(reindexing.get()); } } @Override public void onDone(CompletionCode code, String message) { super.onDone(code, message); - phaser.arriveAndAwaitAdvance(); // Synchronize with the reindex thread. + phaser.arriveAndAwaitAdvance(); // Synchronize with the reindexer control thread. } }; - VisitorParameters parameters = createParameters(type, status.progress().orElse(null)); + VisitorParameters parameters = createParameters(type, status.get().progress().orElse(null)); parameters.setControlHandler(control); - Runnable sessionShutdown = visitorSessions.apply(parameters); + Runnable sessionShutdown = visitorSessions.apply(parameters); // Also starts the visitor session. - // Wait until done; or until termination is forced, in which case we abort the visit and wait for it to complete. - phaser.arriveAndAwaitAdvance(); // Synchronize with the visitor completion thread. - sessionShutdown.run(); + // Wait until done; or until termination is forced, in which we shut down the visitor session immediately. + phaser.arriveAndAwaitAdvance(); // Synchronize with visitor completion. + sessionShutdown.run(); // Shutdown aborts the session, then waits for it to terminate normally. - // If we were interrupted, the result may not yet be set in the control handler. switch (control.getResult().getCode()) { default: log.log(WARNING, "Unexpected visitor result '" + control.getResult().getCode() + "'"); - case FAILURE: // Intentional fallthrough — this is an error. + case FAILURE: // Intentional fallthrough — this is an error. log.log(WARNING, "Visiting failed: " + control.getResult().getMessage()); - status = status.failed(clock.instant(), control.getResult().getMessage()); + status.updateAndGet(value -> value.failed(clock.instant(), control.getResult().getMessage())); break; case ABORTED: - log.log(FINE, () -> "Halting reindexing of " + type + " due to shutdown — will continue later"); - status = status.halted(); + log.log(FINE, () -> "Halting reindexing of " + type + " due to shutdown — will continue later"); + status.updateAndGet(Status::halted); break; case SUCCESS: - log.log(INFO, "Completed reindexing of " + type + " after " + Duration.between(status.startedAt(), clock.instant())); - status = status.successful(clock.instant()); + log.log(INFO, "Completed reindexing of " + type + " after " + Duration.between(status.get().startedAt(), clock.instant())); + status.updateAndGet(value -> value.successful(clock.instant())); } - database.writeReindexing(reindexing.with(type, status)); + database.writeReindexing(reindexing.updateAndGet(value -> value.with(type, status.get()))); + metrics.dump(reindexing.get()); } VisitorParameters createParameters(DocumentType type, ProgressToken progress) { diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexing.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexing.java index 792889e4aa8..51322c37a7d 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexing.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexing.java @@ -121,7 +121,7 @@ public class Reindexing { public Status failed(Instant now, String message) { if (state != State.RUNNING) throw new IllegalStateException("Current state must be RUNNING when changing to FAILED"); - return new Status(startedAt, requireNonNull(now), null, State.FAILED, requireNonNull(message)); + return new Status(startedAt, requireNonNull(now), progress, State.FAILED, requireNonNull(message)); } public Instant startedAt() { diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java index 2044e6869f6..202bd92d86e 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java @@ -28,14 +28,6 @@ import static java.util.stream.Collectors.toUnmodifiableMap; */ public class ReindexingCurator { - private static final String STATUS = "status"; - private static final String TYPE = "type"; - private static final String STARTED_MILLIS = "startedMillis"; - private static final String ENDED_MILLIS = "endedMillis"; - private static final String PROGRESS = "progress"; - private static final String STATE = "state"; - private static final String MESSAGE = "message"; - private final Curator curator; private final String clusterName; private final ReindexingSerializer serializer; @@ -78,6 +70,14 @@ public class ReindexingCurator { private static class ReindexingSerializer { + private static final String STATUS = "status"; + private static final String TYPE = "type"; + private static final String STARTED_MILLIS = "startedMillis"; + private static final String ENDED_MILLIS = "endedMillis"; + private static final String PROGRESS = "progress"; + private static final String STATE = "state"; + private static final String MESSAGE = "message"; + private final DocumentTypeManager types; public ReindexingSerializer(DocumentTypeManager types) { diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java index 740a04619d1..7989338c406 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMaintainer.java @@ -12,6 +12,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.jdisc.Metric; import com.yahoo.net.HostName; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.reindexing.ReindexingConfig; @@ -52,22 +53,23 @@ public class ReindexingMaintainer extends AbstractComponent { @Inject public ReindexingMaintainer(@SuppressWarnings("unused") VespaZooKeeperServer ensureZkHasStarted, + Metric metric, DocumentAccess access, ZookeepersConfig zookeepersConfig, ClusterListConfig clusterListConfig, AllClustersBucketSpacesConfig allClustersBucketSpacesConfig, - ReindexingConfig reindexingConfig, DocumentmanagerConfig documentmanagerConfig) { - this(Clock.systemUTC(), access, zookeepersConfig, clusterListConfig, allClustersBucketSpacesConfig, reindexingConfig, documentmanagerConfig); + ReindexingConfig reindexingConfig) { + this(Clock.systemUTC(), metric, access, zookeepersConfig, clusterListConfig, allClustersBucketSpacesConfig, reindexingConfig); } - ReindexingMaintainer(Clock clock, DocumentAccess access, ZookeepersConfig zookeepersConfig, + ReindexingMaintainer(Clock clock, Metric metric, DocumentAccess access, ZookeepersConfig zookeepersConfig, ClusterListConfig clusterListConfig, AllClustersBucketSpacesConfig allClustersBucketSpacesConfig, - ReindexingConfig reindexingConfig, DocumentmanagerConfig documentmanagerConfig) { - DocumentTypeManager manager = new DocumentTypeManager(documentmanagerConfig); - this.reindexer = new Reindexer(parseCluster(reindexingConfig.clusterName(), clusterListConfig, allClustersBucketSpacesConfig, manager), - parseReady(reindexingConfig, manager), + ReindexingConfig reindexingConfig) { + this.reindexer = new Reindexer(parseCluster(reindexingConfig.clusterName(), clusterListConfig, allClustersBucketSpacesConfig, access.getDocumentTypeManager()), + parseReady(reindexingConfig, access.getDocumentTypeManager()), new ReindexingCurator(Curator.create(zookeepersConfig.zookeeperserverlist()), reindexingConfig.clusterName(), - manager), + access.getDocumentTypeManager()), access, + metric, clock); this.executor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("reindexer-")); if (reindexingConfig.enabled()) diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMetrics.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMetrics.java new file mode 100644 index 00000000000..5e536d1f2ee --- /dev/null +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingMetrics.java @@ -0,0 +1,48 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.reindexing; + +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.jdisc.Metric; + +import java.time.Clock; +import java.util.Map; + +import static ai.vespa.reindexing.Reindexing.State.SUCCESSFUL; + +/** + * Metrics for reindexing in a content cluster. + * + * @author jonmv + */ +class ReindexingMetrics { + + private final Metric metric; + private final String cluster; + + ReindexingMetrics(Metric metric, String cluster) { + this.metric = metric; + this.cluster = cluster; + } + + void dump(Reindexing reindexing) { + reindexing.status().forEach((type, status) -> { + metric.set("reindexing.progress", + status.progress().map(ProgressToken::percentFinished).map(percentage -> percentage * 1e-2) + .orElse(status.state() == SUCCESSFUL ? 1.0 : 0.0), + metric.createContext(Map.of("clusterid", cluster, + "documenttype", type.getName(), + "state", toString(status.state())))); + }); + } + + private static String toString(Reindexing.State state) { + switch (state) { + case READY: return "pending"; + case RUNNING: return "running"; + case FAILED: return "failed"; + case SUCCESSFUL: return "successful"; + default: throw new IllegalArgumentException("Unknown reindexing state '" + state + "'"); + } + } + +} diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java new file mode 100644 index 00000000000..fca08f7743c --- /dev/null +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/http/ReindexingV1ApiHandler.java @@ -0,0 +1,98 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.reindexing.http; + +import ai.vespa.reindexing.Reindexing; +import ai.vespa.reindexing.ReindexingCurator; +import com.google.inject.Inject; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.ZookeepersConfig; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.jdisc.Metric; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.Path; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; +import com.yahoo.vespa.config.content.reindexing.ReindexingConfig; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.zookeeper.VespaZooKeeperServer; + +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; + +/** + * Allows inspecting reindexing status over HTTP. + * + * @author jonmv + */ +public class ReindexingV1ApiHandler extends ThreadedHttpRequestHandler { + + private final ReindexingCurator database; + + @Inject + public ReindexingV1ApiHandler(Executor executor, Metric metric, + @SuppressWarnings("unused") VespaZooKeeperServer ensureZkHasStarted, ZookeepersConfig zookeepersConfig, + ReindexingConfig reindexingConfig, DocumentmanagerConfig documentmanagerConfig) { + this(executor, + metric, + new ReindexingCurator(Curator.create(zookeepersConfig.zookeeperserverlist()), + reindexingConfig.clusterName(), + new DocumentTypeManager(documentmanagerConfig))); + } + + ReindexingV1ApiHandler(Executor executor, Metric metric, ReindexingCurator database) { + super(executor, metric); + this.database = database; + } + + @Override + public HttpResponse handle(HttpRequest request) { + Path path = new Path(request.getUri()); + if (request.getMethod() != GET) + return ErrorResponse.methodNotAllowed("Only GET is supported under /reindexing/v1/"); + + if (path.matches("/reindexing/v1")) return getRoot(); + if (path.matches("/reindexing/v1/status")) return getStatus(); + + return ErrorResponse.notFoundError("Nothing at " + request.getUri().getRawPath()); + } + + HttpResponse getRoot() { + Slime slime = new Slime(); + slime.setObject().setArray("resources").addObject().setString("url", "/reindexing/v1/status"); + return new SlimeJsonResponse(slime); + } + + HttpResponse getStatus() { + Slime slime = new Slime(); + Cursor statusArray = slime.setObject().setArray("status"); + database.readReindexing().status().forEach((type, status) -> { + Cursor statusObject = statusArray.addObject(); + statusObject.setString("type", type.getName()); + statusObject.setLong("startedMillis", status.startedAt().toEpochMilli()); + status.endedAt().ifPresent(endedAt -> statusObject.setLong("endedMillis", endedAt.toEpochMilli())); + status.progress().ifPresent(progress -> statusObject.setString("progress", progress.serializeToString())); + statusObject.setString("state", toString(status.state())); + status.message().ifPresent(message -> statusObject.setString("message", message)); + }); + return new SlimeJsonResponse(slime); + } + + + private static String toString(Reindexing.State state) { + switch (state) { + case READY: return "pending"; + case RUNNING: return "running"; + case SUCCESSFUL: return "successful"; + case FAILED: return "failed"; + default: throw new IllegalArgumentException("Unexpected state '" + state + "'"); + } + } + +} diff --git a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java index a5ad2ba32f1..b0ffdf8ae60 100644 --- a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java +++ b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java @@ -4,7 +4,6 @@ package ai.vespa.reindexing; import ai.vespa.reindexing.Reindexer.Cluster; import ai.vespa.reindexing.Reindexing.Status; import ai.vespa.reindexing.ReindexingCurator.ReindexingLockException; -import com.yahoo.document.Document; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; @@ -12,6 +11,7 @@ import com.yahoo.documentapi.ProgressToken; import com.yahoo.documentapi.VisitorControlHandler; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.jdisc.test.MockMetric; import com.yahoo.searchdefinition.derived.Deriver; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; @@ -40,13 +40,13 @@ import static org.junit.jupiter.api.Assertions.fail; */ class ReindexerTest { - static final Function<VisitorParameters, Runnable> failIfCalled = __ -> () -> { fail("Not supposed to run"); }; + static final Function<VisitorParameters, Runnable> failIfCalled = __ -> () -> fail("Not supposed to run"); final DocumentmanagerConfig musicConfig = Deriver.getDocumentManagerConfig("src/test/resources/schemas/music.sd").build(); final DocumentTypeManager manager = new DocumentTypeManager(musicConfig); final DocumentType music = manager.getDocumentType("music"); - final Document document1 = new Document(music, "id:ns:music::one"); final Cluster cluster = new Cluster("cluster", "id", Map.of(music, "default")); + final MockMetric metric = new MockMetric(); final ManualClock clock = new ManualClock(Instant.EPOCH); ReindexingCurator database; @@ -63,12 +63,13 @@ class ReindexerTest { Map.of(music, Instant.EPOCH), database, failIfCalled, + metric, clock)); } @Test void throwsWhenLockHeldElsewhere() throws InterruptedException, ExecutionException { - Reindexer reindexer = new Reindexer(cluster, Map.of(music, Instant.EPOCH), database, failIfCalled, clock); + Reindexer reindexer = new Reindexer(cluster, Map.of(music, Instant.EPOCH), database, failIfCalled, metric, clock); Executors.newSingleThreadExecutor().submit(database::lockReindexing).get(); assertThrows(ReindexingLockException.class, reindexer::reindex); } @@ -76,12 +77,13 @@ class ReindexerTest { @Test @Timeout(10) void nothingToDoWithEmptyConfig() throws ReindexingLockException { - new Reindexer(cluster, Map.of(), database, failIfCalled, clock).reindex(); + new Reindexer(cluster, Map.of(), database, failIfCalled, metric, clock).reindex(); + assertEquals(Map.of(), metric.metrics()); } @Test void testParameters() { - Reindexer reindexer = new Reindexer(cluster, Map.of(), database, failIfCalled, clock); + Reindexer reindexer = new Reindexer(cluster, Map.of(), database, failIfCalled, metric, clock); ProgressToken token = new ProgressToken(); VisitorParameters parameters = reindexer.createParameters(music, token); assertEquals("music:[document]", parameters.getFieldSet()); @@ -98,14 +100,19 @@ class ReindexerTest { void testReindexing() throws ReindexingLockException { // Reindexer is told to update "music" documents no earlier than EPOCH, which is just now. // Since "music" is a new document type, it is stored as just reindexed, and nothing else happens. - new Reindexer(cluster, Map.of(music, Instant.EPOCH), database, failIfCalled, clock).reindex(); + new Reindexer(cluster, Map.of(music, Instant.EPOCH), database, failIfCalled, metric, clock).reindex(); Reindexing reindexing = Reindexing.empty().with(music, Status.ready(Instant.EPOCH).running().successful(Instant.EPOCH)); assertEquals(reindexing, database.readReindexing()); + assertEquals(Map.of("reindexing.progress", Map.of(Map.of("documenttype", "music", + "clusterid", "cluster", + "state", "successful"), + 1.0)), + metric.metrics()); // New config tells reindexer to reindex "music" documents no earlier than at 10 millis after EPOCH, which isn't yet. // Nothing happens, since it's not yet time. This isn't supposed to happen unless high clock skew. clock.advance(Duration.ofMillis(5)); - new Reindexer(cluster, Map.of(music, Instant.ofEpochMilli(10)), database, failIfCalled, clock).reindex(); + new Reindexer(cluster, Map.of(music, Instant.ofEpochMilli(10)), database, failIfCalled, metric, clock).reindex(); assertEquals(reindexing, database.readReindexing()); // It's time to reindex the "music" documents — let this complete successfully. @@ -116,13 +123,14 @@ class ReindexerTest { database.writeReindexing(Reindexing.empty()); // Wipe database to verify we write data from reindexer. executor.execute(() -> parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.SUCCESS, "OK")); return () -> shutDown.set(true); - }, clock).reindex(); + }, metric, clock).reindex(); reindexing = reindexing.with(music, Status.ready(clock.instant()).running().successful(clock.instant())); assertEquals(reindexing, database.readReindexing()); assertTrue(shutDown.get(), "Session was shut down"); // One more reindexing, this time shut down before visit completes, but after progress is reported. clock.advance(Duration.ofMillis(10)); + metric.metrics().clear(); shutDown.set(false); AtomicReference<Reindexer> aborted = new AtomicReference<>(); aborted.set(new Reindexer(cluster, Map.of(music, Instant.ofEpochMilli(20)), database, parameters -> { @@ -133,11 +141,16 @@ class ReindexerTest { shutDown.set(true); parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.ABORTED, "Shut down"); }; - }, clock)); + }, metric, clock)); aborted.get().reindex(); reindexing = reindexing.with(music, Status.ready(clock.instant()).running().progressed(new ProgressToken()).halted()); assertEquals(reindexing, database.readReindexing()); assertTrue(shutDown.get(), "Session was shut down"); + assertEquals(Map.of("reindexing.progress", Map.of(Map.of("documenttype", "music", + "clusterid", "cluster", + "state", "pending"), + 1.0)), // new ProgressToken() is 100% done. + metric.metrics()); // Last reindexing fails. clock.advance(Duration.ofMillis(10)); @@ -146,13 +159,13 @@ class ReindexerTest { database.writeReindexing(Reindexing.empty()); // Wipe database to verify we write data from reindexer. executor.execute(() -> parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.FAILURE, "Error")); return () -> shutDown.set(true); - }, clock).reindex(); + }, metric, clock).reindex(); reindexing = reindexing.with(music, Status.ready(clock.instant()).running().failed(clock.instant(), "Error")); assertEquals(reindexing, database.readReindexing()); assertTrue(shutDown.get(), "Session was shut down"); // Document type is ignored in next run, as it has failed fatally. - new Reindexer(cluster, Map.of(music, Instant.ofEpochMilli(30)), database, failIfCalled, clock).reindex(); + new Reindexer(cluster, Map.of(music, Instant.ofEpochMilli(30)), database, failIfCalled, metric, clock).reindex(); assertEquals(reindexing, database.readReindexing()); } diff --git a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java new file mode 100644 index 00000000000..1b6379d21e5 --- /dev/null +++ b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java @@ -0,0 +1,80 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.reindexing.http; + +import ai.vespa.reindexing.Reindexing; +import ai.vespa.reindexing.Reindexing.Status; +import ai.vespa.reindexing.ReindexingCurator; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.jdisc.test.MockMetric; +import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.concurrent.Executors; + +import static com.yahoo.jdisc.http.HttpRequest.Method.POST; +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +class ReindexingV1ApiTest { + + DocumentmanagerConfig musicConfig = Deriver.getDocumentManagerConfig("src/test/resources/schemas/music.sd").build(); + DocumentTypeManager manager = new DocumentTypeManager(musicConfig); + DocumentType musicType = manager.getDocumentType("music"); + ReindexingCurator database = new ReindexingCurator(new MockCurator(), "cluster", manager); + ReindexingV1ApiHandler handler = new ReindexingV1ApiHandler(Executors.newSingleThreadExecutor(), new MockMetric(), database); + + @Test + void testResponses() { + RequestHandlerTestDriver driver = new RequestHandlerTestDriver(handler); + + // GET at root + var response = driver.sendRequest("http://localhost/reindexing/v1/"); + assertEquals("{\"resources\":[{\"url\":\"/reindexing/v1/status\"}]}", response.readAll()); + assertEquals("application/json; charset=UTF-8", response.getResponse().headers().getFirst("Content-Type")); + assertEquals(200, response.getStatus()); + + // GET at status with empty database + response = driver.sendRequest("http://localhost/reindexing/v1/status"); + assertEquals("{\"status\":[]}", response.readAll()); + assertEquals(200, response.getStatus()); + + // GET at status with a failed status + database.writeReindexing(Reindexing.empty().with(musicType, Status.ready(Instant.EPOCH) + .running() + .progressed(new ProgressToken()) + .failed(Instant.ofEpochMilli(123), "ヽ(。_°)ノ"))); + response = driver.sendRequest("http://localhost/reindexing/v1/status"); + assertEquals("{\"status\":[{" + + "\"type\":\"music\"," + + "\"startedMillis\":0," + + "\"endedMillis\":123," + + "\"progress\":\"" + new ProgressToken().serializeToString() + "\"," + + "\"state\":\"failed\"," + + "\"message\":\"ヽ(。_°)ノ\"}" + + "]}", + response.readAll()); + assertEquals(200, response.getStatus()); + + // POST at root + response = driver.sendRequest("http://localhost/reindexing/v1/status", POST); + assertEquals("{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Only GET is supported under /reindexing/v1/\"}", + response.readAll()); + assertEquals(405, response.getStatus()); + + // GET at non-existent path + response = driver.sendRequest("http://localhost/reindexing/v1/moo"); + assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"Nothing at /reindexing/v1/moo\"}", + response.readAll()); + assertEquals(404, response.getStatus()); + + } + +} diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 9df68821454..fd229b35778 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -529,15 +529,14 @@ "public static final enum com.yahoo.config.application.api.ValidationId indexModeChange", "public static final enum com.yahoo.config.application.api.ValidationId fieldTypeChange", "public static final enum com.yahoo.config.application.api.ValidationId clusterSizeReduction", + "public static final enum com.yahoo.config.application.api.ValidationId tensorTypeChange", "public static final enum com.yahoo.config.application.api.ValidationId resourcesReduction", "public static final enum com.yahoo.config.application.api.ValidationId contentTypeRemoval", "public static final enum com.yahoo.config.application.api.ValidationId contentClusterRemoval", "public static final enum com.yahoo.config.application.api.ValidationId deploymentRemoval", - "public static final enum com.yahoo.config.application.api.ValidationId skipAutomaticTenantUpgradeTests", "public static final enum com.yahoo.config.application.api.ValidationId globalDocumentChange", "public static final enum com.yahoo.config.application.api.ValidationId configModelVersionMismatch", "public static final enum com.yahoo.config.application.api.ValidationId skipOldConfigModels", - "public static final enum com.yahoo.config.application.api.ValidationId forceAutomaticTenantUpgradeTests", "public static final enum com.yahoo.config.application.api.ValidationId accessControl", "public static final enum com.yahoo.config.application.api.ValidationId globalEndpointChange" ] @@ -577,10 +576,7 @@ "attributes": [ "public" ], - "methods": [ - "public com.yahoo.config.application.api.ValidationId validationId()", - "public java.lang.String getMessage()" - ], + "methods": [], "fields": [] }, "com.yahoo.config.application.api.ValidationOverrides": { @@ -591,6 +587,7 @@ ], "methods": [ "public void <init>(java.util.List)", + "public void invalid(java.util.Map, java.time.Instant)", "public void invalid(com.yahoo.config.application.api.ValidationId, java.lang.String, java.time.Instant)", "public boolean allows(java.lang.String, java.time.Instant)", "public boolean allows(com.yahoo.config.application.api.ValidationId, java.time.Instant)", diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java index c0bae137b0d..4c76d42a17e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java @@ -14,15 +14,14 @@ public enum ValidationId { indexModeChange("indexing-mode-change"), // Changing the index mode (streaming, indexed, store-only) of documents fieldTypeChange("field-type-change"), // Field type changes clusterSizeReduction("cluster-size-reduction"), // Large reductions in cluster size + tensorTypeChange("tensor-type-change"), // Tensor type change resourcesReduction("resources-reduction"), // Large reductions in node resources contentTypeRemoval("content-type-removal"), // Removal of a data type (causes deletion of all data) contentClusterRemoval("content-cluster-removal"), // Removal (or id change) of content clusters deploymentRemoval("deployment-removal"), // Removal of production zones from deployment.xml - skipAutomaticTenantUpgradeTests("skip-automatic-tenant-upgrade-test"), // Skip platform supplied staging tests globalDocumentChange("global-document-change"), // Changing global attribute for document types in content clusters configModelVersionMismatch("config-model-version-mismatch"), // Internal use skipOldConfigModels("skip-old-config-models"), // Internal use - forceAutomaticTenantUpgradeTests("force-automatic-tenant-upgrade-test"), // Internal use accessControl("access-control"), // Internal use, used in zones where there should be no access-control globalEndpointChange("global-endpoint-change"); // Changing global endpoints diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java index a85e109f731..f22cf0d8c47 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java @@ -14,9 +14,13 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.logging.Level; +import java.util.stream.Collectors; /** * A set of allows which suppresses specific validations in limited time periods. @@ -47,6 +51,14 @@ public class ValidationOverrides { this.xmlForm = xmlForm; } + /** Throws a ValidationException unless all given validation is overridden at this time */ + public void invalid(Map<ValidationId, ? extends Collection<String>> messagesByValidationId, Instant now) { + Map<ValidationId, Collection<String>> disallowed = new HashMap<>(messagesByValidationId); + disallowed.keySet().removeIf(id -> allows(id, now)); + if ( ! disallowed.isEmpty()) + throw new ValidationException(disallowed); + } + /** Throws a ValidationException unless this validation is overridden at this time */ public void invalid(ValidationId validationId, String message, Instant now) { if ( ! allows(validationId, now)) @@ -146,26 +158,21 @@ public class ValidationOverrides { /** * A deployment validation exception. * Deployment validations can be {@link ValidationOverrides overridden} based on their id. - * The purpose of this exception is to model that id as a separate field. */ public static class ValidationException extends IllegalArgumentException { static final long serialVersionUID = 789984668; - private final ValidationId validationId; - private ValidationException(ValidationId validationId, String message) { - super(message); - this.validationId = validationId; + super(validationId + ": " + message + ". " + toAllowMessage(validationId)); } - /** Returns the unique id of this validation, which can be used to {@link ValidationOverrides override} it */ - public ValidationId validationId() { return validationId; } - - /** Returns "validationId: message" */ - @Override - public String getMessage() { - return validationId + ": " + super.getMessage() + ". " + toAllowMessage(validationId); + private ValidationException(Map<ValidationId, Collection<String>> messagesById) { + super(messagesById.entrySet().stream() + .map(messages -> messages.getKey() + ":\n\t" + + String.join("\n\t", messages.getValue()) + "\n" + + toAllowMessage(messages.getKey())) + .collect(Collectors.joining("\n"))); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java index 1248560c931..87a150a6c3c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeAction.java @@ -1,9 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ClusterSpec; import java.util.List; +import java.util.Optional; /** * Contains the action to be performed on the given services to handle a config change @@ -37,12 +39,11 @@ public interface ConfigChangeAction { /** Returns the list of services where the action must be performed */ List<ServiceInfo> getServices(); - /** Returns whether this change should be allowed */ - boolean allowed(); + /** When this is non-empty, validation may fail unless this validation id is allowed by validation overrides. */ + default Optional<ValidationId> validationId() { return Optional.empty(); } /** The id of the cluster that needs this action applied */ - // TODO: Remove this default implementation after October 2020 - default ClusterSpec.Id clusterId() { return null; } + ClusterSpec.Id clusterId(); /** Returns whether this change should be ignored for internal redeploy */ default boolean ignoreForInternalRedeploy() { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRefeedAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRefeedAction.java index c5a7bd030e2..13109088dcd 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRefeedAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRefeedAction.java @@ -1,6 +1,8 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; +import com.yahoo.config.application.api.ValidationId; + /** * Represents an action to re-feed a document type in order to handle a config change. * @@ -12,7 +14,7 @@ public interface ConfigChangeRefeedAction extends ConfigChangeAction { default Type getType() { return Type.REFEED; } /** Returns the name identifying this kind of change, used to identify names which should be allowed */ - String name(); + default String name() { return validationId().orElseThrow().value(); } /** Returns the name of the document type that one must re-feed to handle this config change */ String getDocumentType(); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeReindexAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeReindexAction.java index 085638e31ff..bb714a55f94 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeReindexAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeReindexAction.java @@ -1,6 +1,8 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; +import com.yahoo.config.application.api.ValidationId; + /** * Represents an action to re-index a document type in order to handle a config change. * @@ -11,7 +13,7 @@ public interface ConfigChangeReindexAction extends ConfigChangeAction { @Override default Type getType() { return Type.REINDEX; } /** @return name identifying this kind of change, used to identify names which should be allowed */ - String name(); + default String name() { return validationId().orElseThrow().value(); } /** @return name of the document type that must be re-indexed */ String getDocumentType(); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRestartAction.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRestartAction.java index f178180b6e0..c13399a42f5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRestartAction.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigChangeRestartAction.java @@ -11,8 +11,4 @@ public interface ConfigChangeRestartAction extends ConfigChangeAction { @Override default Type getType() { return Type.RESTART; } - /** Restarts are handled automatically so they are allowed */ - @Override - default boolean allowed() { return true; } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index acd4d705889..2ffc24239f9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -97,7 +97,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private static final long serialVersionUID = 1L; - public static final Logger log = Logger.getLogger(VespaModel.class.getPackage().toString()); + public static final Logger log = Logger.getLogger(VespaModel.class.getName()); private final Version version; private final ConfigModelRepo configModelRepo = new ConfigModelRepo(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index ec8607daaca..9e15db348a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -3,13 +3,16 @@ package com.yahoo.vespa.model.admin.clustercontroller; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.api.Reindexing; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.documentapi.DocumentAccessProvider; import com.yahoo.container.di.config.PlatformBundlesConfig; +import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.vespa.config.content.reindexing.ReindexingConfig; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.component.AccessLogComponent; @@ -28,12 +31,15 @@ import java.util.TreeSet; @RestartConfigs({FleetcontrollerConfig.class, ZookeeperServerConfig.class}) public class ClusterControllerContainer extends Container implements PlatformBundlesConfig.Producer, - ZookeeperServerConfig.Producer + ZookeeperServerConfig.Producer, + ReindexingConfig.Producer { private static final ComponentSpecification CLUSTERCONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-apps"); private static final ComponentSpecification ZOOKEEPER_SERVER_BUNDLE = new ComponentSpecification("zookeeper-server"); + private static final ComponentSpecification REINDEXING_CONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-reindexer"); private final Set<String> bundles = new TreeSet<>(); + private final ReindexingContext reindexingContext; public ClusterControllerContainer( AbstractConfigProducer<?> parent, @@ -42,12 +48,16 @@ public class ClusterControllerContainer extends Container implements boolean isHosted, ReindexingContext reindexingContext) { super(parent, "" + index, index, isHosted); + this.reindexingContext = reindexingContext; + addHandler("clustercontroller-status", "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StatusHandler", - "/clustercontroller-status/*"); + "/clustercontroller-status/*", + CLUSTERCONTROLLER_BUNDLE); addHandler("clustercontroller-state-restapi-v2", "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler", - "/cluster/v2/*"); + "/cluster/v2/*", + CLUSTERCONTROLLER_BUNDLE); if (runStandaloneZooKeeper) { addComponent("clustercontroller-zkrunner", "com.yahoo.vespa.zookeeper.VespaZooKeeperServerImpl", @@ -73,7 +83,7 @@ public class ClusterControllerContainer extends Container implements addFileBundle("clustercontroller-core"); addFileBundle("clustercontroller-utils"); addFileBundle("zookeeper-server"); - configureReindexing(reindexingContext); + configureReindexing(); } @Override @@ -110,15 +120,21 @@ public class ClusterControllerContainer extends Container implements addComponent(new Component<>(createComponentModel(id, className, bundle))); } - private void addHandler(String id, String className, String path) { - addHandler(new Handler(createComponentModel(id, className, CLUSTERCONTROLLER_BUNDLE)), path); + private void addHandler(String id, String className, String path, ComponentSpecification bundle) { + addHandler(new Handler(createComponentModel(id, className, bundle)), path); } - private void configureReindexing(ReindexingContext context) { - if (context != null) { - addFileBundle(ReindexingController.REINDEXING_CONTROLLER_BUNDLE); - addComponent(new ReindexingController(context)); + private void configureReindexing() { + if (reindexingContext != null) { + addFileBundle(REINDEXING_CONTROLLER_BUNDLE.getName()); addComponent(new SimpleComponent(DocumentAccessProvider.class.getName())); + addComponent("reindexing-maintainer", + "ai.vespa.reindexing.ReindexingMaintainer", + REINDEXING_CONTROLLER_BUNDLE); + addHandler("reindexing-status", + "ai.vespa.reindexing.http.ReindexingV1ApiHandler", + "/reindexing/v1/*", + REINDEXING_CONTROLLER_BUNDLE); } } @@ -133,4 +149,20 @@ public class ClusterControllerContainer extends Container implements builder.myid(index()); } + @Override + public void getConfig(ReindexingConfig.Builder builder) { + if (reindexingContext == null) + return; + + builder.clusterName(reindexingContext.contentClusterName()); + builder.enabled(reindexingContext.reindexing().enabled()); + for (NewDocumentType type : reindexingContext.documentTypes()) { + String typeName = type.getFullName().getName(); + reindexingContext.reindexing().status(reindexingContext.contentClusterName(), typeName) + .ifPresent(status -> builder.status(typeName, + new ReindexingConfig.Status.Builder() + .readyAtMillis(status.ready().toEpochMilli()))); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ReindexingController.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ReindexingController.java deleted file mode 100644 index 24909ddbc8d..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ReindexingController.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.clustercontroller; - -import com.yahoo.config.model.api.Reindexing; -import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.vespa.config.content.reindexing.ReindexingConfig; -import com.yahoo.vespa.model.container.component.SimpleComponent; - -import java.util.Collection; - -/** - * @author bjorncs - */ -class ReindexingController extends SimpleComponent implements ReindexingConfig.Producer { - - static final String REINDEXING_CONTROLLER_BUNDLE = "clustercontroller-reindexer"; - - private final Reindexing reindexing; - private final String contentClusterName; - private final Collection<NewDocumentType> documentTypes; - - ReindexingController(ReindexingContext context) { - super(new ComponentModel( - BundleInstantiationSpecification.getFromStrings( - "reindexing-maintainer", - "ai.vespa.reindexing.ReindexingMaintainer", - REINDEXING_CONTROLLER_BUNDLE))); - this.reindexing = context.reindexing(); - this.contentClusterName = context.contentClusterName(); - this.documentTypes = context.documentTypes(); - } - - @Override - public void getConfig(ReindexingConfig.Builder builder) { - builder.clusterName(contentClusterName); - builder.enabled(reindexing.enabled()); - for (NewDocumentType type : documentTypes) { - String typeName = type.getFullName().getName(); - reindexing.status(contentClusterName, typeName).ifPresent(status -> - builder.status( - typeName, - new ReindexingConfig.Status.Builder() - .readyAtMillis(status.ready().toEpochMilli()))); - } - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 21f323fc0f3..4317f947e12 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -239,6 +239,8 @@ public class VespaMetricSet { // DO NOT RELY ON THIS METRIC YET. metrics.add(new Metric("cluster-controller.node-event.count")); + metrics.add(new Metric("cluster-controller.reindexing.progress.last")); + return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index d300e31c3dc..e028eaea3c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.Model; @@ -26,13 +27,20 @@ import com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDe import java.time.Instant; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toCollection; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; /** * Executor of validators. This defines the right order of validator execution. @@ -99,9 +107,17 @@ public class Validation { new ContainerRestartValidator(), new NodeResourceChangeValidator() }; - return Arrays.stream(validators) - .flatMap(v -> v.validate(currentModel, nextModel, overrides, now).stream()) - .collect(toList()); + List<ConfigChangeAction> actions = Arrays.stream(validators) + .flatMap(v -> v.validate(currentModel, nextModel, overrides, now).stream()) + .collect(toList()); + + Map<ValidationId, Collection<String>> disallowableActions = actions.stream() + .filter(action -> action.validationId().isPresent()) + .collect(groupingBy(action -> action.validationId().orElseThrow(), + mapping(ConfigChangeAction::getMessage, + toCollection(LinkedHashSet::new)))); + overrides.invalid(disallowableActions, now); + return actions; } private static void validateFirstTimeDeployment(VespaModel model, DeployState deployState) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java index b321d5f3fd7..58cea8c23e5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java @@ -13,7 +13,10 @@ import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; import java.time.Instant; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java index e3f16baf95a..385a678d452 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java @@ -46,22 +46,22 @@ public class IndexingModeChangeValidator implements ChangeValidator { ContentSearchCluster currentSearchCluster = currentCluster.getSearch(); ContentSearchCluster nextSearchCluster = nextCluster.getSearch(); findDocumentTypesWithActionableIndexingModeChange( - actions, overrides, nextCluster, now, + actions, nextCluster, toDocumentTypeNames(currentSearchCluster.getDocumentTypesWithStreamingCluster()), toDocumentTypeNames(nextSearchCluster.getDocumentTypesWithIndexedCluster()), "streaming", "indexed"); findDocumentTypesWithActionableIndexingModeChange( - actions, overrides, nextCluster, now, + actions, nextCluster, toDocumentTypeNames(currentSearchCluster.getDocumentTypesWithIndexedCluster()), toDocumentTypeNames(nextSearchCluster.getDocumentTypesWithStreamingCluster()), "indexed", "streaming"); findDocumentTypesWithActionableIndexingModeChange( - actions, overrides, nextCluster, now, + actions, nextCluster, toDocumentTypeNames(currentSearchCluster.getDocumentTypesWithStoreOnly()), toDocumentTypeNames(nextSearchCluster.getDocumentTypesWithIndexedCluster()), "store-only", "indexed"); findDocumentTypesWithActionableIndexingModeChange( - actions, overrides, nextCluster, now, + actions, nextCluster, toDocumentTypeNames(currentSearchCluster.getDocumentTypesWithIndexedCluster()), toDocumentTypeNames(nextSearchCluster.getDocumentTypesWithStoreOnly()), "indexed", "store-only"); @@ -69,7 +69,7 @@ public class IndexingModeChangeValidator implements ChangeValidator { } private static void findDocumentTypesWithActionableIndexingModeChange( - List<ConfigChangeAction> actions, ValidationOverrides overrides, ContentCluster nextCluster, Instant now, + List<ConfigChangeAction> actions, ContentCluster nextCluster, Set<String> currentTypes, Set<String> nextTypes, String currentIndexMode, String nextIndexingMode) { for (String type : nextTypes) { if (currentTypes.contains(type)) { @@ -78,14 +78,13 @@ public class IndexingModeChangeValidator implements ChangeValidator { .collect(Collectors.toList()); actions.add(VespaReindexAction.of( nextCluster.id(), - ValidationId.indexModeChange.value(), - overrides, + ValidationId.indexModeChange, String.format( "Document type '%s' in cluster '%s' changed indexing mode from '%s' to '%s'", type, nextCluster.getName(), currentIndexMode, nextIndexingMode), services, - type, - now)); + type + )); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index d85d9bd2db5..2f13caa4e09 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -78,7 +78,7 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { NewDocumentType nextDocType, ValidationOverrides overrides, Instant now) { - return new DocumentTypeChangeValidator(id, currentDocType, nextDocType).validate(overrides, now); + return new DocumentTypeChangeValidator(id, currentDocType, nextDocType).validate(); } private static NewDocumentType getDocumentType(ContentCluster cluster, StreamingSearchCluster streamingCluster) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java index 19c63431c03..6a335447a31 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeRefeedAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.application.api.ValidationOverrides; @@ -8,6 +9,7 @@ import com.yahoo.config.provision.ClusterSpec; import java.time.Instant; import java.util.List; +import java.util.Optional; /** * Represents an action to re-feed a document type in order to handle a config change. @@ -22,45 +24,39 @@ public class VespaRefeedAction extends VespaConfigChangeAction implements Config * the validation ids belong to the Vespa model while these names are exposed to the config server, * which is model version independent. */ - private final String name; - + private final ValidationId validationId; private final String documentType; - private final boolean allowed; - private VespaRefeedAction(ClusterSpec.Id id, String name, String message, List<ServiceInfo> services, String documentType, boolean allowed) { + private VespaRefeedAction(ClusterSpec.Id id, ValidationId validationId, String message, List<ServiceInfo> services, String documentType) { super(id, message, services); - this.name = name; + this.validationId = validationId; this.documentType = documentType; - this.allowed = allowed; } /** Creates a refeed action with some missing information */ // TODO: We should require document type or model its absence properly - public static VespaRefeedAction of(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, Instant now) { - return new VespaRefeedAction(id, name, message, List.of(), "", overrides.allows(name, now)); + public static VespaRefeedAction of(ClusterSpec.Id id, ValidationId validationId, String message) { + return new VespaRefeedAction(id, validationId, message, List.of(), ""); } /** Creates a refeed action */ - public static VespaRefeedAction of(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, - List<ServiceInfo> services, String documentType, Instant now) { - return new VespaRefeedAction(id, name, message, services, documentType, overrides.allows(name, now)); + public static VespaRefeedAction of(ClusterSpec.Id id, ValidationId validationId, String message, + List<ServiceInfo> services, String documentType) { + return new VespaRefeedAction(id, validationId, message, services, documentType); } @Override public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) { - return new VespaRefeedAction(clusterId(), name, newMessage, newServices, documentType, allowed); + return new VespaRefeedAction(clusterId(), validationId, newMessage, newServices, documentType); } @Override - public String name() { return name; } + public Optional<ValidationId> validationId() { return Optional.of(validationId); } @Override public String getDocumentType() { return documentType; } @Override - public boolean allowed() { return allowed; } - - @Override public boolean ignoreForInternalRedeploy() { return false; } @@ -76,14 +72,13 @@ public class VespaRefeedAction extends VespaConfigChangeAction implements Config if ( ! (o instanceof VespaRefeedAction)) return false; VespaRefeedAction other = (VespaRefeedAction)o; if ( ! this.documentType.equals(other.documentType)) return false; - if ( ! this.name.equals(other.name)) return false; - if ( ! this.allowed == other.allowed) return false; + if ( ! this.validationId.equals(other.validationId)) return false; return true; } @Override public int hashCode() { - return 31 * super.hashCode() + 11 * name.hashCode() + documentType.hashCode(); + return 31 * super.hashCode() + 11 * validationId.hashCode() + documentType.hashCode(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java index f10802afc31..8b4060e7d19 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java @@ -1,14 +1,14 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; -import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeReindexAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; -import java.time.Instant; import java.util.List; import java.util.Objects; +import java.util.Optional; /** * Represents an action to re-index a document type in order to handle a config change. @@ -22,36 +22,32 @@ public class VespaReindexAction extends VespaConfigChangeAction implements Confi * the validation ids belong to the Vespa model while these names are exposed to the config server, * which is model version independent. */ - private final String name; + private final ValidationId validationId; private final String documentType; - private final boolean allowed; - private VespaReindexAction(ClusterSpec.Id id, String name, String message, List<ServiceInfo> services, String documentType, boolean allowed) { + private VespaReindexAction(ClusterSpec.Id id, ValidationId validationId, String message, List<ServiceInfo> services, String documentType) { super(id, message, services); - this.name = name; + this.validationId = validationId; this.documentType = documentType; - this.allowed = allowed; } - public static VespaReindexAction of( - ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, Instant now) { - return new VespaReindexAction(id, name, message, List.of(), /*documentType*/null, overrides.allows(name, now)); + public static VespaReindexAction of(ClusterSpec.Id id, ValidationId validationId, String message) { + return new VespaReindexAction(id, validationId, message, List.of(), /*documentType*/null); } public static VespaReindexAction of( - ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, - List<ServiceInfo> services, String documentType, Instant now) { - return new VespaReindexAction(id, name, message, services, documentType, overrides.allows(name, now)); + ClusterSpec.Id id, ValidationId validationId, String message, + List<ServiceInfo> services, String documentType) { + return new VespaReindexAction(id, validationId, message, services, documentType); } @Override public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) { - return new VespaReindexAction(clusterId(), name, newMessage, newServices, documentType, allowed); + return new VespaReindexAction(clusterId(), validationId, newMessage, newServices, documentType); } - @Override public String name() { return name; } + @Override public Optional<ValidationId> validationId() { return Optional.of(validationId); } @Override public String getDocumentType() { return documentType; } - @Override public boolean allowed() { return allowed; } @Override public boolean ignoreForInternalRedeploy() { return false; } @Override public String toString() { return super.toString() + ", documentType='" + documentType + "'"; } @@ -61,10 +57,10 @@ public class VespaReindexAction extends VespaConfigChangeAction implements Confi if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; VespaReindexAction that = (VespaReindexAction) o; - return allowed == that.allowed && - Objects.equals(name, that.name) && - Objects.equals(documentType, that.documentType); + return Objects.equals(validationId, that.validationId) && + Objects.equals(documentType, that.documentType); } - @Override public int hashCode() { return Objects.hash(super.hashCode(), name, documentType, allowed); } + @Override public int hashCode() { return Objects.hash(super.hashCode(), validationId, documentType); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java index a10aac30298..0aee0675ea7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java @@ -6,13 +6,10 @@ import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.IndexSchema; import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.searchdefinition.document.HnswIndexParams; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; -import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; -import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -51,12 +48,11 @@ public class AttributeChangeValidator { this.nextDocType = nextDocType; } - public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { + public List<VespaConfigChangeAction> validate() { List<VespaConfigChangeAction> result = new ArrayList<>(); result.addAll(validateAddAttributeAspect()); result.addAll(validateRemoveAttributeAspect()); result.addAll(validateAttributeSettings()); - result.addAll(validateTensorTypes(overrides, now)); return result; } @@ -144,36 +140,4 @@ public class AttributeChangeValidator { } } - private List<VespaConfigChangeAction> validateTensorTypes(ValidationOverrides overrides, Instant now) { - List<VespaConfigChangeAction> result = new ArrayList<>(); - - for (Attribute nextAttr : nextFields.attributes()) { - Attribute currentAttr = currentFields.getAttribute(nextAttr.getName()); - - if (currentAttr != null && currentAttr.tensorType().isPresent()) { - // If the tensor attribute is not present on the new attribute, it means that the data type of the attribute - // has been changed. This is already handled by DocumentTypeChangeValidator, so we can ignore it here - if (!nextAttr.tensorType().isPresent()) { - continue; - } - - // Tensor attribute has changed type - if (!nextAttr.tensorType().get().equals(currentAttr.tensorType().get())) { - result.add(createTensorTypeChangedRefeedAction(id, currentAttr, nextAttr, overrides, now)); - } - } - } - - return result; - } - - private static VespaRefeedAction createTensorTypeChangedRefeedAction(ClusterSpec.Id id, Attribute currentAttr, Attribute nextAttr, ValidationOverrides overrides, Instant now) { - return VespaRefeedAction.of(id, - "tensor-type-change", - overrides, - new ChangeMessageBuilder(nextAttr.getName()).addChange("tensor type", - currentAttr.tensorType().get().toString(), - nextAttr.tensorType().get().toString()).build(), now); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java index 68a97f33dfd..ce435a4c157 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java @@ -38,20 +38,20 @@ public class DocumentDatabaseChangeValidator { public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { List<VespaConfigChangeAction> result = new ArrayList<>(); - result.addAll(validateAttributeChanges(overrides, now)); + result.addAll(validateAttributeChanges()); result.addAll(validateStructFieldAttributeChanges(overrides, now)); result.addAll(validateIndexingScriptChanges(overrides, now)); result.addAll(validateDocumentTypeChanges(overrides, now)); return result; } - private List<VespaConfigChangeAction> validateAttributeChanges(ValidationOverrides overrides, Instant now) { + private List<VespaConfigChangeAction> validateAttributeChanges() { return new AttributeChangeValidator(id, currentDatabase.getDerivedConfiguration().getAttributeFields(), currentDatabase.getDerivedConfiguration().getIndexSchema(), currentDocType, nextDatabase.getDerivedConfiguration().getAttributeFields(), nextDatabase.getDerivedConfiguration().getIndexSchema(), nextDocType) - .validate(overrides, now); + .validate(); } private List<VespaConfigChangeAction> validateStructFieldAttributeChanges(ValidationOverrides overrides, Instant now) { @@ -72,7 +72,7 @@ public class DocumentDatabaseChangeValidator { private List<VespaConfigChangeAction> validateDocumentTypeChanges(ValidationOverrides overrides, Instant now) { return new DocumentTypeChangeValidator(id, currentDocType, nextDocType) - .validate(overrides, now); + .validate(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java index b66145a10c5..5b7fdfad0f7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java @@ -1,15 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.document.Field; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; -import java.time.Instant; import java.util.List; import java.util.stream.Collectors; @@ -136,17 +135,16 @@ public class DocumentTypeChangeValidator { this.nextDocType = nextDocType; } - public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { + public List<VespaConfigChangeAction> validate() { return currentDocType.getAllFields().stream(). map(field -> createFieldChange(field, nextDocType)). filter(fieldChange -> fieldChange.valid() && fieldChange.changedType()). map(fieldChange -> VespaRefeedAction.of(id, - "field-type-change", - overrides, + ValidationId.fieldTypeChange, new ChangeMessageBuilder(fieldChange.fieldName()). addChange("data type", fieldChange.currentTypeName(), - fieldChange.nextTypeName()).build(), - now)). + fieldChange.nextTypeName()).build() + )). collect(Collectors.toList()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java index 8f9b1a3ed77..e3f3abf0747 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -41,21 +41,20 @@ public class IndexingScriptChangeValidator { String fieldName = nextField.getName(); ImmutableSDField currentField = currentSearch.getConcreteField(fieldName); if (currentField != null) { - validateScripts(currentField, nextField, overrides, now).ifPresent(r -> result.add(r)); + validateScripts(currentField, nextField).ifPresent(r -> result.add(r)); } } return result; } - private Optional<VespaConfigChangeAction> validateScripts(ImmutableSDField currentField, ImmutableSDField nextField, - ValidationOverrides overrides, Instant now) { + private Optional<VespaConfigChangeAction> validateScripts(ImmutableSDField currentField, ImmutableSDField nextField) { ScriptExpression currentScript = currentField.getIndexingScript(); ScriptExpression nextScript = nextField.getIndexingScript(); if ( ! equalScripts(currentScript, nextScript)) { ChangeMessageBuilder messageBuilder = new ChangeMessageBuilder(nextField.getName()); new IndexingScriptChangeMessageBuilder(currentSearch, currentField, nextSearch, nextField).populate(messageBuilder); messageBuilder.addChange("indexing script", currentScript.toString(), nextScript.toString()); - return Optional.of(VespaReindexAction.of(id, ValidationId.indexingChange.value(), overrides, messageBuilder.build(), now)); + return Optional.of(VespaReindexAction.of(id, ValidationId.indexingChange, messageBuilder.build())); } return Optional.empty(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 1e5c944f8dc..00ff19ae68a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -3,6 +3,10 @@ package com.yahoo.vespa.model.container.http; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.BindingPattern; @@ -10,6 +14,7 @@ import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.http.ssl.HostedSslConnectorFactory; import java.util.Collection; import java.util.Collections; @@ -26,11 +31,13 @@ import java.util.Set; */ public class AccessControl { - public enum ClientAuthentication { want, need } + + public enum ClientAuthentication { want, need;} public static final ComponentId ACCESS_CONTROL_CHAIN_ID = ComponentId.fromString("access-control-chain"); - public static final ComponentId ACCESS_CONTROL_EXCLUDED_CHAIN_ID = ComponentId.fromString("access-control-excluded-chain"); + public static final ComponentId ACCESS_CONTROL_EXCLUDED_CHAIN_ID = ComponentId.fromString("access-control-excluded-chain"); + public static final ComponentId DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID = ComponentId.fromString("default-connector-hosted-request-chain"); private static final int HOSTED_CONTAINER_PORT = 4443; // Handlers that are excluded from access control @@ -43,7 +50,6 @@ public class AccessControl { ApplicationContainerCluster.METRICS_V2_HANDLER_CLASS, ApplicationContainerCluster.PROMETHEUS_V1_HANDLER_CLASS ); - public static class Builder { private final String domain; private boolean readEnabled = false; @@ -51,7 +57,6 @@ public class AccessControl { private ClientAuthentication clientAuthentication = ClientAuthentication.need; private final Set<BindingPattern> excludeBindings = new LinkedHashSet<>(); private Collection<Handler<?>> handlers = Collections.emptyList(); - public Builder(String domain) { this.domain = domain; } @@ -111,9 +116,26 @@ public class AccessControl { http.setAccessControl(this); addAccessControlFilterChain(http); addAccessControlExcludedChain(http); + addDefaultHostedRequestChain(http); removeDuplicateBindingsFromAccessControlChain(http); } + public void configureHostedConnector(HostedSslConnectorFactory connectorFactory) { + connectorFactory.setDefaultRequestFilterChain(ACCESS_CONTROL_CHAIN_ID); + } + + public void configureDefaultHostedConnector(Http http) { + // Set default filter chain on local port + http.getHttpServer() + .get() + .getConnectorFactories() + .stream() + .filter(cf -> cf.getListenPort() == Defaults.getDefaults().vespaWebServicePort()) + .findFirst() + .orElseThrow(() -> new RuntimeException("Could not find default connector")) + .setDefaultRequestFilterChain(DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID); + } + /** returns the excluded bindings as specified in 'access-control' in services.xml **/ public Set<BindingPattern> excludedBindings() { return excludedBindings; } @@ -127,7 +149,6 @@ public class AccessControl { private void addAccessControlFilterChain(Http http) { http.getFilterChains().add(createChain(ACCESS_CONTROL_CHAIN_ID)); - http.getBindings().addAll(List.of(createAccessControlBinding("/"), createAccessControlBinding("/*"))); } private void addAccessControlExcludedChain(Http http) { @@ -144,9 +165,14 @@ public class AccessControl { } } + // Add a filter chain used by default hosted connector + private void addDefaultHostedRequestChain(Http http) { + Chain<Filter> chain = createChain(DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID); + http.getFilterChains().add(chain); + } + // Remove bindings from access control chain that have binding pattern as a different filter chain private void removeDuplicateBindingsFromAccessControlChain(Http http) { - removeDuplicateBindingsFromChain(http, ACCESS_CONTROL_CHAIN_ID); removeDuplicateBindingsFromChain(http, ACCESS_CONTROL_EXCLUDED_CHAIN_ID); } @@ -172,13 +198,6 @@ public class AccessControl { } - private static FilterBinding createAccessControlBinding(String path) { - return FilterBinding.create( - FilterBinding.Type.REQUEST, - new ComponentSpecification(ACCESS_CONTROL_CHAIN_ID.stringValue()), - SystemBindingPattern.fromHttpPortAndPath(Integer.toString(HOSTED_CONTAINER_PORT), path)); - } - private static FilterBinding createAccessControlExcludedBinding(BindingPattern excludedBinding) { BindingPattern rewrittenBinding = SystemBindingPattern.fromHttpPortAndPath( Integer.toString(HOSTED_CONTAINER_PORT), excludedBinding.path()); // only keep path from excluded binding diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java index 1ed043857e2..b5c3cac1879 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -24,6 +24,7 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl private final List<FilterBinding> bindings = new CopyOnWriteArrayList<>(); private volatile JettyHttpServer httpServer; private volatile AccessControl accessControl; + private volatile boolean strictFiltering = false; // TODO Vespa 8: Enable strict filtering by default if filtering is enabled public Http(FilterChains chains) { super("http"); @@ -72,6 +73,8 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl return Optional.ofNullable(accessControl); } + public void setStrictFiltering(boolean enabled) { this.strictFiltering = enabled; } + @Override public void getConfig(ServerConfig.Builder builder) { for (FilterBinding binding : bindings) { @@ -80,6 +83,7 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl .binding(binding.binding().patternString())); } populateDefaultFiltersConfig(builder, httpServer); + builder.strictFiltering(strictFiltering); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index 9b9ebedda6d..5417a522d6a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -40,11 +40,14 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> FilterChains filterChains; List<FilterBinding> bindings = new ArrayList<>(); AccessControl accessControl = null; + Optional<Boolean> strictFiltering = Optional.empty(); Element filteringElem = XML.getChild(spec, "filtering"); if (filteringElem != null) { filterChains = new FilterChainsBuilder().build(deployState, ancestor, filteringElem); bindings = readFilterBindings(filteringElem); + strictFiltering = XmlHelper.getOptionalAttribute(filteringElem, "strict-mode") + .map(Boolean::valueOf); Element accessControlElem = XML.getChild(filteringElem, "access-control"); if (accessControlElem != null) { @@ -55,6 +58,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> } Http http = new Http(filterChains); + strictFiltering.ifPresent(http::setStrictFiltering); http.getBindings().addAll(bindings); ApplicationContainerCluster cluster = getContainerCluster(ancestor).orElse(null); http.setHttpServer(new JettyHttpServerBuilder(cluster).build(deployState, ancestor, spec)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index c8f2bd08ea5..7eea5d8496f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -85,6 +85,7 @@ import org.w3c.dom.Node; import java.net.URI; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -318,15 +319,22 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (isHostedTenantApplication(context)) { addHostedImplicitHttpIfNotPresent(cluster); addHostedImplicitAccessControlIfNotPresent(deployState, cluster); + addDefaultConnectorHostedFilterBinding(cluster); addAdditionalHostedConnector(deployState, cluster, context); } } + private void addDefaultConnectorHostedFilterBinding(ApplicationContainerCluster cluster) { + cluster.getHttp().getAccessControl() + .ifPresent(accessControl -> accessControl.configureDefaultHostedConnector(cluster.getHttp())); ; + } + private void addAdditionalHostedConnector(DeployState deployState, ApplicationContainerCluster cluster, ConfigModelContext context) { JettyHttpServer server = cluster.getHttp().getHttpServer().get(); String serverName = server.getComponentId().getName(); // If the deployment contains certificate/private key reference, setup TLS port + HostedSslConnectorFactory connectorFactory; if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { @@ -340,13 +348,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .map(clientAuth -> clientAuth.equals(AccessControl.ClientAuthentication.need)) .orElse(false); - HostedSslConnectorFactory connectorFactory = authorizeClient + connectorFactory = authorizeClient ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets, enforceHandshakeClientAuth); - server.addConnector(connectorFactory); } else { - server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName)); + connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName); } + cluster.getHttp().getAccessControl().ifPresent(accessControl -> accessControl.configureHostedConnector(connectorFactory)); + server.addConnector(connectorFactory); } private static boolean isHostedTenantApplication(ConfigModelContext context) { @@ -359,10 +368,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if(cluster.getHttp() == null) { cluster.setHttp(new Http(new FilterChains(cluster))); } - if(cluster.getHttp().getHttpServer().isEmpty()) { - JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"), cluster, cluster.isHostedVespa()); - cluster.getHttp().setHttpServer(defaultHttpServer); - defaultHttpServer.addConnector(new ConnectorFactory.Builder("SearchServer", Defaults.getDefaults().vespaWebServicePort()).build()); + JettyHttpServer httpServer = cluster.getHttp().getHttpServer().orElse(null); + if (httpServer == null) { + httpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"), cluster, cluster.isHostedVespa()); + cluster.getHttp().setHttpServer(httpServer); + } + int defaultPort = Defaults.getDefaults().vespaWebServicePort(); + boolean defaultConnectorPresent = httpServer.getConnectorFactories().stream().anyMatch(connector -> connector.getListenPort() == defaultPort); + if (!defaultConnectorPresent) { + httpServer.addConnector(new ConnectorFactory.Builder("SearchServer", defaultPort).build()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java index f3f9022f6f0..2157839ef5c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import java.time.Instant; @@ -24,26 +24,22 @@ public class ConfigChangeTestUtils { return new VespaRestartAction(id, message, services); } - public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, String name, String message) { - return VespaRefeedAction.of(id, name, ValidationOverrides.empty, message, Instant.now()); + public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, ValidationId validationId, String message) { + return VespaRefeedAction.of(id, validationId, message); } - public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, Instant now) { - return VespaRefeedAction.of(id, name, overrides, message, now); + public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, ValidationId validationId, String message, + List<ServiceInfo> services, String documentType) { + return VespaRefeedAction.of(id, validationId, message, services, documentType); } - public static VespaConfigChangeAction newRefeedAction(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, - List<ServiceInfo> services, String documentType, Instant now) { - return VespaRefeedAction.of(id, name, overrides, message, services, documentType, now); + public static VespaConfigChangeAction newReindexAction(ClusterSpec.Id id, ValidationId validationId, String message) { + return VespaReindexAction.of(id, validationId, message); } - public static VespaConfigChangeAction newReindexAction(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, Instant now) { - return VespaReindexAction.of(id, name, overrides, message, now); - } - - public static VespaConfigChangeAction newReindexAction(ClusterSpec.Id id, String name, ValidationOverrides overrides, String message, - List<ServiceInfo> services, String documentType, Instant now) { - return VespaReindexAction.of(id, name, overrides, message, services, documentType, now); + public static VespaConfigChangeAction newReindexAction(ClusterSpec.Id id, ValidationId validationId, String message, + List<ServiceInfo> services, String documentType) { + return VespaReindexAction.of(id, validationId, message, services, documentType); } public static List<ConfigChangeAction> normalizeServicesInActions(List<ConfigChangeAction> result) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java index 8d365f24c7f..2b211c561d9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; @@ -146,9 +147,8 @@ public class IndexedSearchClusterChangeValidatorTest { public void requireThatChangingFieldTypeIsDiscovered() { Fixture f = Fixture.newOneDocFixture(STRING_FIELD, INT_FIELD); f.assertValidation(List.of(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Document type 'd1': " + FIELD_TYPE_CHANGE_MSG, FOO_SERVICE, "d1", Instant.now()))); + ValidationId.fieldTypeChange, + "Document type 'd1': " + FIELD_TYPE_CHANGE_MSG, FOO_SERVICE, "d1"))); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java index 8e2cecf89b4..ba9dfcdc388 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationOverrides.ValidationException; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ConfigChangeReindexAction; import com.yahoo.config.provision.Environment; @@ -11,8 +12,10 @@ import org.junit.Test; import java.util.List; import java.util.stream.Collectors; +import static java.util.stream.Collectors.joining; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author bratseth @@ -21,6 +24,25 @@ import static org.junit.Assert.assertTrue; public class IndexingModeChangeValidatorTest { @Test + public void testChangingIndexModeFromIndexedToStreamingWhenDisallowed() { + ValidationTester tester = new ValidationTester(); + + VespaModel oldModel = + tester.deploy(null, getServices("index"), Environment.prod, "<validation-overrides />").getFirst(); + try { + List<ConfigChangeAction> changeActions = + tester.deploy(oldModel, getServices("streaming"), Environment.prod, "<calidation-overrides />").getSecond(); + fail("Should throw on disallowed config change action"); + } + catch (ValidationException e) { + assertEquals("indexing-mode-change:\n" + + "\tDocument type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'streaming'\n" + + "To allow this add <allow until='yyyy-mm-dd'>indexing-mode-change</allow> to validation-overrides.xml, see https://docs.vespa.ai/documentation/reference/validation-overrides.html", + e.getMessage()); + } + } + + @Test public void testChangingIndexModeFromIndexedToStreaming() { ValidationTester tester = new ValidationTester(); @@ -29,9 +51,9 @@ public class IndexingModeChangeValidatorTest { List<ConfigChangeAction> changeActions = tester.deploy(oldModel, getServices("streaming"), Environment.prod, validationOverrides).getSecond(); - assertReindexingChange(true, // allowed=true due to validation override - "Document type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'streaming'", - changeActions); + assertReindexingChange( // allowed=true due to validation override + "Document type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'streaming'", + changeActions); } @Test @@ -43,23 +65,22 @@ public class IndexingModeChangeValidatorTest { List<ConfigChangeAction> changeActions = tester.deploy(oldModel, getServices("store-only"), Environment.prod, validationOverrides).getSecond(); - assertReindexingChange(true, // allowed=true due to validation override - "Document type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'store-only'", - changeActions); + assertReindexingChange( // allowed=true due to validation override + "Document type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'store-only'", + changeActions); } - private void assertReindexingChange(boolean allowed, String message, List<ConfigChangeAction> changeActions) { + private void assertReindexingChange(String message, List<ConfigChangeAction> changeActions) { List<ConfigChangeAction> reindexingActions = changeActions.stream() .filter(a -> a instanceof ConfigChangeReindexAction) .collect(Collectors.toList()); assertEquals(1, reindexingActions.size()); - assertEquals(allowed, reindexingActions.get(0).allowed()); assertTrue(reindexingActions.get(0) instanceof ConfigChangeReindexAction); assertEquals("indexing-mode-change", ((ConfigChangeReindexAction)reindexingActions.get(0)).name()); assertEquals(message, reindexingActions.get(0).getMessage()); } - private static final String getServices(String indexingMode) { + private static String getServices(String indexingMode) { return "<services version='1.0'>" + " <content id='default' version='1.0'>" + " <redundancy>1</redundancy>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java index f5ef50ee3a4..18aac032fe7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; @@ -164,10 +165,9 @@ public class StreamingSearchClusterChangeValidatorTest { private static VespaConfigChangeAction createFieldTypeChangeRefeedAction(String docType, List<ServiceInfo> service) { return ConfigChangeTestUtils.newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Document type '" + docType + "': Field 'f1' changed: data type: 'string' -> 'int'", - service, docType, Instant.now()); + ValidationId.fieldTypeChange, + "Document type '" + docType + "': Field 'f1' changed: data type: 'string' -> 'int'", + service, docType); } private static VespaConfigChangeAction createAddFastAccessRestartAction() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java index 168ee797fbf..e89f0c0a9cd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java @@ -1,15 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import org.junit.Test; -import java.time.Instant; import java.util.List; -import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction; import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRestartAction; public class AttributeChangeValidatorTest { @@ -30,7 +27,7 @@ public class AttributeChangeValidatorTest { @Override public List<VespaConfigChangeAction> validate() { - return validator.validate(ValidationOverrides.empty, Instant.now()); + return validator.validate(); } } @@ -111,33 +108,6 @@ public class AttributeChangeValidatorTest { } @Test - public void changing_tensor_type_of_tensor_field_requires_refeed() throws Exception { - new Fixture( - "field f1 type tensor(x[2]) { indexing: attribute }", - "field f1 type tensor(x[3]) { indexing: attribute }") - .assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "tensor-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: tensor type: 'tensor(x[2])' -> 'tensor(x[3])'", Instant.now())); - - new Fixture( - "field f1 type tensor(x[5]) { indexing: attribute }", - "field f1 type tensor(x[3]) { indexing: attribute }") - .assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "tensor-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: tensor type: 'tensor(x[5])' -> 'tensor(x[3])'", Instant.now())); - } - - @Test - public void not_changing_tensor_type_of_tensor_field_is_ok() throws Exception { - new Fixture( - "field f1 type tensor(x[2]) { indexing: attribute }", - "field f1 type tensor(x[2]) { indexing: attribute }") - .assertValidation(); - } - - @Test public void adding_rank_filter_requires_restart() throws Exception { new Fixture("field f1 type string { indexing: attribute }", "field f1 type string { indexing: attribute \n rank: filter }"). diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java index a4fbf474a7f..1f64d41e371 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; @@ -47,20 +48,17 @@ public class DocumentDatabaseChangeValidatorTest { "field f2 type string { indexing: index | summary } " + "field f3 type string { indexing: summary } " + "field f4 type array<s> { struct-field s1 { indexing: attribute } }"); + Instant.now(); f.assertValidation(Arrays.asList( newRestartAction(ClusterSpec.Id.from("test"), "Field 'f1' changed: add attribute aspect"), newRestartAction(ClusterSpec.Id.from("test"), "Field 'f4.s1' changed: add attribute aspect"), newReindexAction(ClusterSpec.Id.from("test"), - "indexing-change", - ValidationOverrides.empty, - "Field 'f2' changed: add index aspect, indexing script: '{ input f2 | summary f2; }' -> " + - "'{ input f2 | tokenize normalize stem:\"BEST\" | index f2 | summary f2; }'", Instant.now()), - newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f3' changed: data type: 'int' -> 'string'", Instant.now()))); + ValidationId.indexingChange, + "Field 'f2' changed: add index aspect, indexing script: '{ input f2 | summary f2; }' -> " + + "'{ input f2 | tokenize normalize stem:\"BEST\" | index f2 | summary f2; }'"), + newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f3' changed: data type: 'int' -> 'string'"))); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 190c2c8c645..8ee2a924503 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; @@ -8,7 +9,6 @@ import com.yahoo.document.ReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.searchdefinition.FieldSets; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; import org.junit.Test; @@ -42,7 +42,7 @@ public class DocumentTypeChangeValidatorTest { @Override public List<VespaConfigChangeAction> validate() { - return validator.validate(ValidationOverrides.empty, Instant.now()); + return validator.validate(); } } @@ -65,21 +65,16 @@ public class DocumentTypeChangeValidatorTest { public void requireThatDataTypeChangeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: summary }", "field f1 type int { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'string' -> 'int'", - Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'string' -> 'int'")); } @Test public void requireThatAddingCollectionTypeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: summary }", "field f1 type array<string> { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'string' -> 'Array<string>'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'string' -> 'Array<string>'")); } @@ -94,34 +89,26 @@ public class DocumentTypeChangeValidatorTest { public void requireThatNestedDataTypeChangeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type array<string> { indexing: summary }", "field f1 type array<int> { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'Array<string>' -> 'Array<int>'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'Array<string>' -> 'Array<int>'")); } @Test public void requireThatChangedCollectionTypeIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type array<string> { indexing: summary }", "field f1 type weightedset<string> { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'Array<string>' -> 'WeightedSet<string>'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'Array<string>' -> 'WeightedSet<string>'")); } @Test public void requireThatMultipleDataTypeChangesIsNotOK() throws Exception { Fixture f = new Fixture("field f1 type string { indexing: summary } field f2 type int { indexing: summary }" , "field f2 type string { indexing: summary } field f1 type int { indexing: summary }"); - f.assertValidation(Arrays.asList(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f1' changed: data type: 'string' -> 'int'", Instant.now()), - newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f2' changed: data type: 'int' -> 'string'", Instant.now()))); + Instant.now(); + Instant.now(); + f.assertValidation(Arrays.asList(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'string' -> 'int'"), + newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f2' changed: data type: 'int' -> 'string'"))); } @Test @@ -156,40 +143,32 @@ public class DocumentTypeChangeValidatorTest { public void requireThatDataTypeChangeInStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type string {} } field f2 type s1 { indexing: summary }", "struct s1 { field f1 type int {} } field f2 type s1 { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f2' changed: data type: 's1:{f1:string}' -> 's1:{f1:int}'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f2' changed: data type: 's1:{f1:string}' -> 's1:{f1:int}'")); } @Test public void requireThatNestedDataTypeChangeInStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type array<string> {} } field f2 type s1 { indexing: summary }", "struct s1 { field f1 type array<int> {} } field f2 type s1 { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f2' changed: data type: 's1:{f1:Array<string>}' -> 's1:{f1:Array<int>}'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f2' changed: data type: 's1:{f1:Array<string>}' -> 's1:{f1:Array<int>}'")); } @Test public void requireThatDataTypeChangeInNestedStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type string {} } struct s2 { field f2 type s1 {} } field f3 type s2 { indexing: summary }", "struct s1 { field f1 type int {} } struct s2 { field f2 type s1 {} } field f3 type s2 { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f3' changed: data type: 's2:{s1:{f1:string}}' -> 's2:{s1:{f1:int}}'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f3' changed: data type: 's2:{s1:{f1:string}}' -> 's2:{s1:{f1:int}}'")); } @Test public void requireThatMultipleDataTypeChangesInStructFieldIsNotOK() throws Exception { Fixture f = new Fixture("struct s1 { field f1 type string {} field f2 type int {} } field f3 type s1 { indexing: summary }", "struct s1 { field f1 type int {} field f2 type string {} } field f3 type s1 { indexing: summary }"); - f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), - "field-type-change", - ValidationOverrides.empty, - "Field 'f3' changed: data type: 's1:{f1:string,f2:int}' -> 's1:{f1:int,f2:string}'", Instant.now())); + Instant.now(); + f.assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f3' changed: data type: 's1:{f1:string,f2:int}' -> 's1:{f1:int,f2:string}'")); } @Test @@ -197,7 +176,7 @@ public class DocumentTypeChangeValidatorTest { var validator = new DocumentTypeChangeValidator(ClusterSpec.Id.from("test"), createDocumentTypeWithReferenceField("oldDoc"), createDocumentTypeWithReferenceField("newDoc")); - List<VespaConfigChangeAction> result = validator.validate(ValidationOverrides.empty, Instant.now()); + List<VespaConfigChangeAction> result = validator.validate(); assertEquals(1, result.size()); VespaConfigChangeAction action = result.get(0); assertTrue(action instanceof VespaRefeedAction); @@ -208,6 +187,21 @@ public class DocumentTypeChangeValidatorTest { action.toString()); } + @Test + public void changing_tensor_type_of_tensor_field_requires_refeed() throws Exception { + Instant.now(); + new Fixture( + "field f1 type tensor(x[2]) { indexing: attribute }", + "field f1 type tensor(x[3]) { indexing: attribute }") + .assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'tensor(x[2])' -> 'tensor(x[3])'")); + + Instant.now(); + new Fixture( + "field f1 type tensor(x[5]) { indexing: attribute }", + "field f1 type tensor(x[3]) { indexing: attribute }") + .assertValidation(newRefeedAction(ClusterSpec.Id.from("test"), ValidationId.fieldTypeChange, "Field 'f1' changed: data type: 'tensor(x[5])' -> 'tensor(x[3])'")); + } + private static NewDocumentType createDocumentTypeWithReferenceField(String nameReferencedDocumentType) { StructDataType headerfields = new StructDataType("headerfields"); headerfields.addField(new Field("ref", new ReferenceDataType(new DocumentType(nameReferencedDocumentType), 0))); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index 9f418476a24..2e1ec53f886 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; @@ -56,12 +57,10 @@ public class IndexingScriptChangeValidatorTest { private static VespaConfigChangeAction expectedReindexingAction(String field, String changedMsg, String fromScript, String toScript) { return VespaReindexAction.of(ClusterSpec.Id.from("test"), - "indexing-change", - ValidationOverrides.empty, - "Field '" + field + "' changed: " + + ValidationId.indexingChange, + "Field '" + field + "' changed: " + (changedMsg.isEmpty() ? "" : changedMsg + ", ") + - "indexing script: '" + fromScript + "' -> '" + toScript + "'", - Instant.now()); + "indexing script: '" + fromScript + "' -> '" + toScript + "'"); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java index 04efffab438..0bc4ecbfdfd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java @@ -36,7 +36,7 @@ public class StructFieldAttributeChangeValidatorTestCase { public List<VespaConfigChangeAction> validate() { List<VespaConfigChangeAction> result = new ArrayList<>(); result.addAll(structFieldAttributeValidator.validate(ValidationOverrides.empty, Instant.now())); - result.addAll(docTypeValidator.validate(ValidationOverrides.empty, Instant.now())); + result.addAll(docTypeValidator.validate()); return result; } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java index 62a36422dd8..7d4be4b5e33 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerIncludeTest.java @@ -18,7 +18,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @since 5.1.13 */ public class ContainerIncludeTest { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/StrictFilteringTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/StrictFilteringTest.java new file mode 100644 index 00000000000..98383e77324 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/StrictFilteringTest.java @@ -0,0 +1,41 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.jdisc.http.ServerConfig; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import org.junit.Test; +import org.w3c.dom.Element; + +import static org.junit.Assert.assertTrue; + +/** + * @author bjorncs + */ +public class StrictFilteringTest extends DomBuilderTest { + + @Test + public void default_request_and_response_filters_in_services_xml_are_listen_in_server_config() { + Element xml = parse( + "<container version='1.0'>", + " <http>", + " <filtering strict-mode=\"true\">", + " <request-chain id='request-chain-with-binding'>", + " <filter id='my-filter' class='MyFilter'/>", + " <binding>http://*/my-chain-binding</binding>", + " </request-chain>", + " </filtering>", + " <server id='server1' port='8000' />", + " </http>", + "</container>"); + buildContainerCluster(xml); + ServerConfig config = root.getConfig(ServerConfig.class, "container/http/jdisc-jetty/server1"); + assertTrue(config.strictFiltering()); + } + + private void buildContainerCluster(Element containerElem) { + new ContainerModelBuilder(true, ContainerModelBuilder.Networking.enable).build(DeployState.createTestState(), null, null, root, containerElem); + root.freezeModelTopology(); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index f21ab28be72..4993a51ab74 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -6,10 +6,13 @@ import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.http.AccessControl; +import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; +import com.yahoo.vespa.model.container.http.ssl.HostedSslConnectorFactory; import org.junit.Test; import java.util.ArrayList; @@ -22,6 +25,7 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.not; @@ -48,6 +52,7 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { FilterChains filterChains = http.getFilterChains(); assertTrue(filterChains.hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID)); assertTrue(filterChains.hasChain(AccessControl.ACCESS_CONTROL_EXCLUDED_CHAIN_ID)); + assertTrue(filterChains.hasChain(AccessControl.DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID)); } @Test @@ -152,15 +157,24 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { } @Test - public void access_control_filter_chain_contains_catchall_bindings() { + public void hosted_connector_for_port_4443_uses_access_control_filter_chain_as_default_request_filter_chain() { Http http = createModelAndGetHttp( " <http>", " <filtering>", " <access-control/>", " </filtering>", " </http>"); + Set<String> actualBindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_CHAIN_ID); - assertThat(actualBindings, containsInAnyOrder("http://*:4443/*")); + assertThat(actualBindings, empty()); + + HostedSslConnectorFactory hostedConnectorFactory = (HostedSslConnectorFactory)http.getHttpServer().get().getConnectorFactories().stream() + .filter(connectorFactory -> connectorFactory instanceof HostedSslConnectorFactory) + .findAny() + .get(); + Optional<ComponentId> maybeDefaultChain = hostedConnectorFactory.getDefaultRequestFilterChain(); + assertTrue(maybeDefaultChain.isPresent()); + assertEquals(AccessControl.ACCESS_CONTROL_CHAIN_ID, maybeDefaultChain.get()); } @Test @@ -193,7 +207,7 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { } @Test - public void access_control_chains_does_not_contain_duplicate_bindings_to_user_request_filter_chain() { + public void access_control_chain_exclude_chain_does_not_contain_duplicate_bindings_to_user_request_filter_chain() { Http http = createModelAndGetHttp( " <http>", " <handler id='custom.Handler'>", @@ -221,9 +235,6 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { "http://*:4443/metrics/v2", "http://*:4443/metrics/v2/*")); - Set<String> actualAccessControlBindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_CHAIN_ID); - assertThat(actualAccessControlBindings, containsInAnyOrder("http://*:4443/*")); - Set<String> actualCustomChainBindings = getFilterBindings(http, ComponentId.fromString("my-custom-request-chain")); assertThat(actualCustomChainBindings, containsInAnyOrder("http://*/custom-handler/*", "http://*/")); } @@ -261,9 +272,6 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { "http://*:4443/", "http://*:4443/custom-handler/*")); - Set<String> actualAccessControlBindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_CHAIN_ID); - assertThat(actualAccessControlBindings, containsInAnyOrder("http://*:4443/*")); - Set<String> actualCustomChainBindings = getFilterBindings(http, ComponentId.fromString("my-custom-response-chain")); assertThat(actualCustomChainBindings, containsInAnyOrder("http://*/custom-handler/*")); } @@ -292,6 +300,28 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { assertEquals(AccessControl.ClientAuthentication.want, http.getAccessControl().get().clientAuthentication); } + @Test + public void local_connector_has_default_chain() { + Http http = createModelAndGetHttp( + " <http>", + " <filtering>", + " <access-control/>", + " </filtering>", + " </http>"); + + Set<String> actualBindings = getFilterBindings(http, AccessControl.DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID); + assertThat(actualBindings, empty()); + + ConnectorFactory connectorFactory = http.getHttpServer().get().getConnectorFactories().stream() + .filter(cf -> cf.getListenPort() == Defaults.getDefaults().vespaWebServicePort()) + .findAny() + .get(); + + Optional<ComponentId> defaultChain = connectorFactory.getDefaultRequestFilterChain(); + assertTrue(defaultChain.isPresent()); + assertEquals(AccessControl.DEFAULT_CONNECTOR_HOSTED_REQUEST_CHAIN_ID, defaultChain.get()); + } + private Http createModelAndGetHttp(String... httpElement) { List<String> servicesXml = new ArrayList<>(); servicesXml.add("<container version='1.0'>"); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java index f1c86485a64..8f4c9f81d7f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java @@ -1,6 +1,8 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import com.yahoo.config.Node; + import java.util.List; import java.util.Objects; @@ -53,6 +55,14 @@ public class ClusterResources { return true; } + /** Returns the total resources of this, that is the number of nodes times the node resources */ + public NodeResources totalResources() { + return nodeResources.withVcpu(nodeResources.vcpu() * nodes) + .withMemoryGb(nodeResources.memoryGb() * nodes) + .withDiskGb(nodeResources.diskGb() * nodes) + .withBandwidthGbps(nodeResources.bandwidthGbps() * nodes); + } + @Override public boolean equals(Object o) { if (o == this) return true; diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java index cea40da52b9..f79abbddc56 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigVerification.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; import ai.vespa.util.http.VespaHttpClientBuilder; @@ -72,23 +72,22 @@ public class ConfigVerification { for (Map.Entry<String, Stack<String>> entry : mappings.entrySet()) { recurseUrls.add(entry.getValue().pop()); } - int ret = compareOutputs(performRequests(recurseUrls, httpClient)); - if (ret != 0) { - return ret; - } + if ( ! equalOutputs(performRequests(recurseUrls, httpClient))) + return -1; } return 0; } - private static int compareOutputs(Map<String, String> outputs) { + private static boolean equalOutputs(Map<String, String> outputs) { Map.Entry<String, String> firstEntry = outputs.entrySet().iterator().next(); for (Map.Entry<String, String> entry : outputs.entrySet()) { if (!entry.getValue().equals(firstEntry.getValue())) { - System.out.println("output from '" + entry.getKey() + "' did not equal output from '" + firstEntry.getKey() + "'"); - return -1; + System.out.println("output from '" + entry.getKey() + "': '" + entry.getValue() + + "' did not equal output from '" + firstEntry.getKey() + "': '" + firstEntry.getValue() + "'"); + return false; } } - return 0; + return true; } private static String performRequest(String url, CloseableHttpClient httpClient) throws IOException { diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index 33c568061fe..f22f5e5cb1c 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -7,6 +7,7 @@ namespace=cloud.config # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true +tenants{}.applications{}.usePowerOfTwoChoicesLb bool default=false tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index b356b99c50a..6cc05a0f69e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -326,12 +326,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); Deployment deployment = prepare(sessionId, prepareParams, logger); - if (deployment.configChangeActions().getRefeedActions().getEntries().stream().anyMatch(entry -> ! entry.allowed())) - logger.log(Level.WARNING, "Activation rejected because of disallowed re-feed actions"); - else if (deployment.configChangeActions().getReindexActions().getEntries().stream().anyMatch(entry -> ! entry.allowed())) - logger.log(Level.WARNING, "Activation rejected because of disallowed re-index actions"); - else - deployment.activate(); + deployment.activate(); return new PrepareResult(sessionId, deployment.configChangeActions(), logger); } @@ -1014,21 +1009,19 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye RestartActions restartActions = actions.getRestartActions(); if ( ! restartActions.isEmpty()) { logger.log(Level.WARNING, "Change(s) between active and new application that require restart:\n" + - restartActions.format()); + restartActions.format()); } RefeedActions refeedActions = actions.getRefeedActions(); if ( ! refeedActions.isEmpty()) { - boolean allAllowed = refeedActions.getEntries().stream().allMatch(RefeedActions.Entry::allowed); - logger.log(allAllowed ? Level.INFO : Level.WARNING, + logger.log(Level.WARNING, "Change(s) between active and new application that may require re-feed:\n" + - refeedActions.format()); + refeedActions.format()); } ReindexActions reindexActions = actions.getReindexActions(); if ( ! reindexActions.isEmpty()) { - boolean allAllowed = reindexActions.getEntries().stream().allMatch(ReindexActions.Entry::allowed); - logger.log(allAllowed ? Level.INFO : Level.WARNING, - "Change(s) between active and new application that may require re-index:\n" + - reindexActions.format()); + logger.log(Level.WARNING, + "Change(s) between active and new application that may require re-index:\n" + + reindexActions.format()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index 2e73a02c75b..8d001d5d5df 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -130,7 +130,13 @@ public class Application implements ModelResult { if (logDebug()) { debug("Resolving " + configKey + " with config definition " + def); } - ConfigPayload payload = model.getConfig(configKey, def); + + ConfigPayload payload = null; + try { + payload = model.getConfig(configKey, def); + } catch (Exception e) { + throw new ConfigurationRuntimeException("Unable to get config for " + app, e); + } if (payload == null) { metricUpdater.incrementFailedRequests(); throw new ConfigurationRuntimeException("Unable to resolve config " + configKey); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverter.java index ec48b671a5b..1a0d109b6c9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverter.java @@ -42,7 +42,6 @@ public class ConfigChangeActionsSlimeConverter { for (RefeedActions.Entry entry : actions.getRefeedActions().getEntries()) { Cursor entryCursor = refeedCursor.addObject(); entryCursor.setString("name", entry.name()); - entryCursor.setBool("allowed", entry.allowed()); entryCursor.setString("documentType", entry.getDocumentType()); entryCursor.setString("clusterName", entry.getClusterName()); messagesToSlime(entryCursor, entry.getMessages()); @@ -55,7 +54,6 @@ public class ConfigChangeActionsSlimeConverter { for (ReindexActions.Entry entry : actions.getReindexActions().getEntries()) { Cursor entryCursor = refeedCursor.addObject(); entryCursor.setString("name", entry.name()); - entryCursor.setBool("allowed", entry.allowed()); entryCursor.setString("documentType", entry.getDocumentType()); entryCursor.setString("clusterName", entry.getClusterName()); messagesToSlime(entryCursor, entry.getMessages()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActions.java index c20b8527f2e..b2221cbcf6c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActions.java @@ -5,7 +5,13 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ConfigChangeRefeedAction; import com.yahoo.config.model.api.ServiceInfo; -import java.util.*; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; /** * Represents all actions to re-feed document types in order to handle config changes. @@ -17,15 +23,13 @@ public class RefeedActions { public static class Entry { private final String name; - private final boolean allowed; private final String documentType; private final String clusterName; private final Set<ServiceInfo> services = new LinkedHashSet<>(); private final Set<String> messages = new TreeSet<>(); - private Entry(String name, boolean allowed, String documentType, String clusterName) { + private Entry(String name, String documentType, String clusterName) { this.name = name; - this.allowed = allowed; this.documentType = documentType; this.clusterName = clusterName; } @@ -42,8 +46,6 @@ public class RefeedActions { public String name() { return name; } - public boolean allowed() { return allowed; } - public String getDocumentType() { return documentType; } public String getClusterName() { return clusterName; } @@ -54,12 +56,12 @@ public class RefeedActions { } - private Entry addEntry(String name, boolean allowed, String documentType, ServiceInfo service) { + private Entry addEntry(String name, String documentType, ServiceInfo service) { String clusterName = service.getProperty("clustername").orElse(""); - String entryId = name + "." + allowed + "." + clusterName + "." + documentType; + String entryId = name + "." + "." + clusterName + "." + documentType; Entry entry = actions.get(entryId); if (entry == null) { - entry = new Entry(name, allowed, documentType, clusterName); + entry = new Entry(name, documentType, clusterName); actions.put(entryId, entry); } return entry; @@ -75,7 +77,7 @@ public class RefeedActions { if (action.getType().equals(ConfigChangeAction.Type.REFEED)) { ConfigChangeRefeedAction refeedAction = (ConfigChangeRefeedAction) action; for (ServiceInfo service : refeedAction.getServices()) { - addEntry(refeedAction.name(), refeedAction.allowed(), refeedAction.getDocumentType(), service). + addEntry(refeedAction.name(), refeedAction.getDocumentType(), service). addService(service). addMessage(action.getMessage()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java index 425276cebd6..6e2e23ab6be 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java @@ -18,8 +18,6 @@ public class RefeedActionsFormatter { public String format() { StringBuilder builder = new StringBuilder(); for (RefeedActions.Entry entry : actions.getEntries()) { - if (entry.allowed()) - builder.append("(allowed) "); builder.append(entry.name() + ": Consider removing data and re-feed document type '" + entry.getDocumentType() + "' in cluster '" + entry.getClusterName() + "' because:\n"); int counter = 1; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActions.java index e328f9595b7..6ed1c43623f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActions.java @@ -27,7 +27,7 @@ public class ReindexActions { if (action.getType().equals(ConfigChangeAction.Type.REINDEX)) { ConfigChangeReindexAction reindexChange = (ConfigChangeReindexAction) action; for (ServiceInfo service : reindexChange.getServices()) { - addEntry(reindexChange.name(), reindexChange.allowed(), reindexChange.getDocumentType(), service). + addEntry(reindexChange.name(), reindexChange.getDocumentType(), service). addService(service). addMessage(action.getMessage()); } @@ -35,12 +35,12 @@ public class ReindexActions { } } - private Entry addEntry(String name, boolean allowed, String documentType, ServiceInfo service) { + private Entry addEntry(String name, String documentType, ServiceInfo service) { String clusterName = service.getProperty("clustername").orElse(""); - String entryId = name + "." + allowed + "." + clusterName + "." + documentType; + String entryId = name + "." + "." + clusterName + "." + documentType; Entry entry = actions.get(entryId); if (entry == null) { - entry = new Entry(name, allowed, documentType, clusterName); + entry = new Entry(name, documentType, clusterName); actions.put(entryId, entry); } return entry; @@ -53,15 +53,13 @@ public class ReindexActions { public static class Entry { private final String name; - private final boolean allowed; private final String documentType; private final String clusterName; private final Set<ServiceInfo> services = new LinkedHashSet<>(); private final Set<String> messages = new TreeSet<>(); - private Entry(String name, boolean allowed, String documentType, String clusterName) { + private Entry(String name, String documentType, String clusterName) { this.name = name; - this.allowed = allowed; this.documentType = documentType; this.clusterName = clusterName; } @@ -77,7 +75,6 @@ public class ReindexActions { } public String name() { return name; } - public boolean allowed() { return allowed; } public String getDocumentType() { return documentType; } public String getClusterName() { return clusterName; } public Set<ServiceInfo> getServices() { return services; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatter.java index e89bfd522cd..bdd01404f64 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatter.java @@ -17,8 +17,6 @@ class ReindexActionsFormatter { String format() { StringBuilder builder = new StringBuilder(); for (ReindexActions.Entry entry : actions.getEntries()) { - if (entry.allowed()) - builder.append("(allowed) "); builder.append(entry.name() + ": Consider re-indexing document type '" + entry.getDocumentType() + "' in cluster '" + entry.getClusterName() + "' because:\n"); int counter = 1; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index a49d98176d2..6fb315dc3b3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; import com.yahoo.vespa.config.server.http.NotFoundException; +import com.yahoo.vespa.config.server.tenant.Tenant; import java.io.IOException; import java.time.Duration; @@ -220,11 +221,11 @@ public class ApplicationHandler extends HttpHandler { } private void triggerReindexing(HttpRequest request, ApplicationId applicationId) { - List<String> clusters = Optional.ofNullable(request.getProperty("cluster")).stream() + List<String> clusters = Optional.ofNullable(request.getProperty("clusterId")).stream() .flatMap(value -> Stream.of(value.split(","))) .filter(cluster -> ! cluster.isBlank()) .collect(toList()); - List<String> types = Optional.ofNullable(request.getProperty("type")).stream() + List<String> types = Optional.ofNullable(request.getProperty("documentType")).stream() .flatMap(value -> Stream.of(value.split(","))) .filter(type -> ! type.isBlank()) .collect(toList()); @@ -244,9 +245,13 @@ public class ApplicationHandler extends HttpHandler { } private HttpResponse getReindexingStatus(ApplicationId applicationId) { - return new ReindexResponse(applicationRepository.getTenant(applicationId).getApplicationRepo().database() - .readReindexingStatus(applicationId) - .orElseThrow(() -> new NotFoundException("Reindexing status not found for " + applicationId))); + Tenant tenant = applicationRepository.getTenant(applicationId); + if (tenant == null) + throw new NotFoundException("Tenant '" + applicationId.tenant().value() + "' not found"); + + return new ReindexResponse(tenant.getApplicationRepo().database() + .readReindexingStatus(applicationId) + .orElseThrow(() -> new NotFoundException("Reindexing status not found for " + applicationId))); } private HttpResponse restart(HttpRequest request, ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java index 19534bba810..556a2a5b8d3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.maintenance; import com.yahoo.log.LogLevel; @@ -17,6 +17,7 @@ import java.time.Duration; */ public class SessionsMaintainer extends ConfigServerMaintainer { private final boolean hostedVespa; + private int iteration = 0; SessionsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, FlagSource flagSource) { super(applicationRepository, curator, flagSource, Duration.ofMinutes(1), interval); @@ -25,6 +26,9 @@ public class SessionsMaintainer extends ConfigServerMaintainer { @Override protected boolean maintain() { + if (iteration % 10 == 0) + log.log(LogLevel.INFO, () -> "Running " + SessionsMaintainer.class.getSimpleName() + ", iteration " + iteration); + applicationRepository.deleteExpiredLocalSessions(); if (hostedVespa) { @@ -33,6 +37,7 @@ public class SessionsMaintainer extends ConfigServerMaintainer { log.log(LogLevel.FINE, () -> "Deleted " + deleted + " expired remote sessions older than " + expiryTime); } + iteration++; return true; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index ae258445e88..d816c3215a7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -9,7 +9,10 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import java.util.Collections; import java.util.Comparator; @@ -32,10 +35,12 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; + private final BooleanFlag usePowerOfTwoChoicesLb; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; + usePowerOfTwoChoicesLb = Flags.USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING.bindTo(flagSource); } @Override @@ -67,6 +72,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) { LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); ab.activeRotation(getActiveRotation(app)); + ab.usePowerOfTwoChoicesLb(usePowerOfTwoChoicesLb(app)); app.getModel().getHosts().stream() .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) .forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo))); @@ -87,6 +93,10 @@ public class LbServicesProducer implements LbServicesConfig.Producer { return activeRotation; } + private boolean usePowerOfTwoChoicesLb(ApplicationInfo app) { + return usePowerOfTwoChoicesLb.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value(); + } + private LbServicesConfig.Tenants.Applications.Hosts.Builder getHostsConfig(HostInfo hostInfo) { LbServicesConfig.Tenants.Applications.Hosts.Builder hb = new LbServicesConfig.Tenants.Applications.Hosts.Builder(); hb.hostname(hostInfo.getHostname()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 8679d7d678a..93c1e4c2b50 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -569,8 +569,10 @@ public class SessionRepository { } private void copyApp(File sourceDir, File destinationDir) throws IOException { - if (destinationDir.exists()) - throw new RuntimeException("Destination dir " + destinationDir + " already exists"); + if (destinationDir.exists()) { + log.log(Level.INFO, "Destination dir " + destinationDir + " already exists, app has already been copied"); + return; + } if (! sourceDir.isDirectory()) throw new IllegalArgumentException(sourceDir.getAbsolutePath() + " is not a directory"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java index b5194432682..876b169742f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java @@ -16,7 +16,6 @@ import java.util.List; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * @author geirst @@ -44,20 +43,16 @@ public class ConfigChangeActionsBuilder { } - ConfigChangeActionsBuilder refeed(String name, boolean allowed, String message, String documentType, String clusterName, String serviceName) { - actions.add(new MockRefeedAction(name, - allowed, + ConfigChangeActionsBuilder refeed(ValidationId validationId, String message, String documentType, String clusterName, String serviceName) { + actions.add(new MockRefeedAction(validationId, message, List.of(createService(clusterName, "myclustertype", "myservicetype", serviceName)), documentType)); return this; } - ConfigChangeActionsBuilder reindex(String name, boolean allowed, String message, String documentType, String clusterName, String serviceName) { + ConfigChangeActionsBuilder reindex(ValidationId validationId, String message, String documentType, String clusterName, String serviceName) { List<ServiceInfo> services = List.of(createService(clusterName, "myclustertype", "myservicetype", serviceName)); - ValidationOverrides overrides = mock(ValidationOverrides.class); - when(overrides.allows((String) any(), any())).thenReturn(allowed); - when(overrides.allows((ValidationId) any(), any())).thenReturn(allowed); - actions.add(VespaReindexAction.of(ClusterSpec.Id.from(clusterName), name, overrides, message, services, documentType, Instant.now())); + actions.add(VespaReindexAction.of(ClusterSpec.Id.from(clusterName), validationId, message, services, documentType)); return this; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverterTest.java index d145a796725..d75f95d4c48 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverterTest.java @@ -89,16 +89,15 @@ public class ConfigChangeActionsSlimeConverterTest { @Test public void json_representation_of_refeed_actions() throws IOException { ConfigChangeActions actions = new ConfigChangeActionsBuilder(). - refeed(CHANGE_ID, true, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_TYPE). - refeed(CHANGE_ID_2, false, CHANGE_MSG, DOC_TYPE_2, CLUSTER, SERVICE_TYPE).build(); + refeed(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_TYPE). + refeed(CHANGE_ID_2, CHANGE_MSG, DOC_TYPE_2, CLUSTER, SERVICE_TYPE).build(); assertEquals("{\n" + " \"configChangeActions\": {\n" + " \"restart\": [\n" + " ],\n" + " \"refeed\": [\n" + " {\n" + - " \"name\": \"change-id\",\n" + - " \"allowed\": true,\n" + + " \"name\": \"field-type-change\",\n" + " \"documentType\": \"music\",\n" + " \"clusterName\": \"foo\",\n" + " \"messages\": [\n" + @@ -114,8 +113,7 @@ public class ConfigChangeActionsSlimeConverterTest { " ]\n" + " },\n" + " {\n" + - " \"name\": \"other-change-id\",\n" + - " \"allowed\": false,\n" + + " \"name\": \"indexing-change\",\n" + " \"documentType\": \"book\",\n" + " \"clusterName\": \"foo\",\n" + " \"messages\": [\n" + @@ -141,7 +139,7 @@ public class ConfigChangeActionsSlimeConverterTest { @Test public void json_representation_of_reindex_actions() throws IOException { ConfigChangeActions actions = new ConfigChangeActionsBuilder(). - reindex(CHANGE_ID, true, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_TYPE).build(); + reindex(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_TYPE).build(); assertEquals( "{\n" + " \"configChangeActions\": {\n" + @@ -151,8 +149,7 @@ public class ConfigChangeActionsSlimeConverterTest { " ],\n" + " \"reindex\": [\n" + " {\n" + - " \"name\": \"change-id\",\n" + - " \"allowed\": true,\n" + + " \"name\": \"field-type-change\",\n" + " \"documentType\": \"music\",\n" + " \"clusterName\": \"foo\",\n" + " \"messages\": [\n" + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java index 11f2a46994c..615d4c86c1d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java @@ -1,32 +1,35 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.configchange; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeRefeedAction; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ClusterSpec; import java.util.List; +import java.util.Optional; /** * @author geirst */ public class MockRefeedAction extends MockConfigChangeAction implements ConfigChangeRefeedAction { - private final String name; - private final boolean allowed; + private final ValidationId validationId; private final String documentType; - public MockRefeedAction(String name, boolean allowed, String message, List<ServiceInfo> services, String documentType) { + public MockRefeedAction(ValidationId validationId, String message, List<ServiceInfo> services, String documentType) { super(message, services); - this.name = name; - this.allowed = allowed; + this.validationId = validationId; this.documentType = documentType; } @Override - public String name() { return name; } + public Optional<ValidationId> validationId() { return Optional.of(validationId); } @Override - public boolean allowed() { return allowed; } + public ClusterSpec.Id clusterId() { + return null; + } @Override public boolean ignoreForInternalRedeploy() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java index 48d6833129e..4b898b501ec 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java @@ -15,9 +15,9 @@ public class RefeedActionsFormatterTest { @Test public void formatting_of_single_action() { RefeedActions actions = new ConfigChangeActionsBuilder(). - refeed(CHANGE_ID, false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). build().getRefeedActions(); - assertEquals("change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + + assertEquals("field-type-change: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + " 1) change\n", new RefeedActionsFormatter(actions).format()); } @@ -25,20 +25,18 @@ public class RefeedActionsFormatterTest { @Test public void formatting_of_multiple_actions() { RefeedActions actions = new ConfigChangeActionsBuilder(). - refeed(CHANGE_ID, false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed(CHANGE_ID, false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed(CHANGE_ID_2, false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed(CHANGE_ID_2, true, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed(CHANGE_ID, false, CHANGE_MSG_2, DOC_TYPE_2, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID_2, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID_2, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID, CHANGE_MSG_2, DOC_TYPE_2, CLUSTER, SERVICE_NAME). build().getRefeedActions(); - assertEquals("change-id: Consider removing data and re-feed document type 'book' in cluster 'foo' because:\n" + + assertEquals("field-type-change: Consider removing data and re-feed document type 'book' in cluster 'foo' because:\n" + " 1) other change\n" + - "change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + + "field-type-change: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + " 1) change\n" + " 2) other change\n" + - "other-change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + - " 1) other change\n" + - "(allowed) other-change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + + "indexing-change: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + " 1) other change\n", new RefeedActionsFormatter(actions).format()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsTest.java index 7235b8905c5..24e81dc3f99 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.configchange; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ServiceInfo; import org.junit.Test; @@ -32,8 +33,8 @@ public class RefeedActionsTest { @Test public void action_with_multiple_reasons() { List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed("change-id", false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). build().getRefeedActions().getEntries(); assertThat(entries.size(), is(1)); assertThat(toString(entries.get(0)), equalTo("music.foo:[baz][change,other change]")); @@ -42,8 +43,8 @@ public class RefeedActionsTest { @Test public void actions_with_multiple_services() { List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME_2). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME_2). build().getRefeedActions().getEntries(); assertThat(entries.size(), is(1)); assertThat(toString(entries.get(0)), equalTo("music.foo:[baz,qux][change]")); @@ -52,8 +53,8 @@ public class RefeedActionsTest { @Test public void actions_with_multiple_document_types() { List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE_2, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE_2, CLUSTER, SERVICE_NAME). build().getRefeedActions().getEntries(); assertThat(entries.size(), is(2)); assertThat(toString(entries.get(0)), equalTo("book.foo:[baz][change]")); @@ -63,8 +64,8 @@ public class RefeedActionsTest { @Test public void actions_with_multiple_clusters() { List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER_2, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(ValidationId.indexModeChange, CHANGE_MSG, DOC_TYPE, CLUSTER_2, SERVICE_NAME). build().getRefeedActions().getEntries(); assertThat(entries.size(), is(2)); assertThat(toString(entries.get(0)), equalTo("music.bar:[baz][change]")); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatterTest.java index e9dd3f3bbfc..b07d002a431 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ReindexActionsFormatterTest.java @@ -21,9 +21,9 @@ public class ReindexActionsFormatterTest { @Test public void formatting_of_single_action() { ReindexActions actions = new ConfigChangeActionsBuilder(). - reindex(CHANGE_ID, false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). build().getReindexActions(); - assertEquals("change-id: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + + assertEquals("field-type-change: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + " 1) change\n", new ReindexActionsFormatter(actions).format()); } @@ -31,20 +31,18 @@ public class ReindexActionsFormatterTest { @Test public void formatting_of_multiple_actions() { ReindexActions actions = new ConfigChangeActionsBuilder(). - reindex(CHANGE_ID, false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). - reindex(CHANGE_ID, false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - reindex(CHANGE_ID_2, false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - reindex(CHANGE_ID_2, true, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). - reindex(CHANGE_ID, false, CHANGE_MSG_2, DOC_TYPE_2, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID_2, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID_2, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + reindex(CHANGE_ID, CHANGE_MSG_2, DOC_TYPE_2, CLUSTER, SERVICE_NAME). build().getReindexActions(); - assertEquals("change-id: Consider re-indexing document type 'book' in cluster 'foo' because:\n" + + assertEquals("field-type-change: Consider re-indexing document type 'book' in cluster 'foo' because:\n" + " 1) other change\n" + - "change-id: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + + "field-type-change: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + " 1) change\n" + " 2) other change\n" + - "other-change-id: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + - " 1) other change\n" + - "(allowed) other-change-id: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + + "indexing-change: Consider re-indexing document type 'music' in cluster 'foo' because:\n" + " 1) other change\n", new ReindexActionsFormatter(actions).format()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/Utils.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/Utils.java index 8499c12f648..e02e1e2b143 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/Utils.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/Utils.java @@ -1,14 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.configchange; +import com.yahoo.config.application.api.ValidationId; + /** * @author geirst * @since 5.44 */ public class Utils { - final static String CHANGE_ID = "change-id"; - final static String CHANGE_ID_2 = "other-change-id"; + final static ValidationId CHANGE_ID = ValidationId.fieldTypeChange; + final static ValidationId CHANGE_ID_2 = ValidationId.indexingChange; final static String CHANGE_MSG = "change"; final static String CHANGE_MSG_2 = "other change"; final static String DOC_TYPE = "music"; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 4d1b9341e7f..341fa7109da 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; -import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; @@ -49,6 +49,7 @@ import static com.yahoo.vespa.config.server.deploy.DeployTester.createFailingMod import static com.yahoo.vespa.config.server.deploy.DeployTester.createHostedModelFactory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -390,27 +391,6 @@ public class HostedDeployTest { } @Test - public void testThatDisallowedConfigChangeActionsBlockDeployment() throws IOException { - List<Host> hosts = List.of(createHost("host1", "6.1.0"), - createHost("host2", "6.1.0"), - createHost("host3", "6.1.0"), - createHost("host4", "6.1.0")); - List<ServiceInfo> services = List.of( - new ServiceInfo("serviceName", "serviceType", null, Map.of("clustername", "cluster"), "configId", "hostName")); - - ManualClock clock = new ManualClock(Instant.EPOCH); - List<ModelFactory> modelFactories = List.of( - new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), - VespaReindexAction.of(ClusterSpec.Id.from("test"), "indexing-mode-change", ValidationOverrides.empty, - "reindex please", services, "music", clock.instant()), - new VespaRestartAction(ClusterSpec.Id.from("test"), "change", services))); - - DeployTester tester = createTester(hosts, modelFactories, prodZone, clock); - PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.1.0"); - assertNull("Deployment was not activated", tester.applicationRepository().getActiveSession(tester.applicationId())); - } - - @Test public void testThatAllowedConfigChangeActionsAreActedUpon() throws IOException { List<Host> hosts = List.of(createHost("host1", "6.1.0"), createHost("host2", "6.1.0"), @@ -422,8 +402,8 @@ public class HostedDeployTest { ManualClock clock = new ManualClock(Instant.EPOCH); List<ModelFactory> modelFactories = List.of( new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), - VespaReindexAction.of(ClusterSpec.Id.from("test"), "indexing-mode-change", ValidationOverrides.all, - "reindex please", services, "music", clock.instant()), + VespaReindexAction.of(ClusterSpec.Id.from("test"), ValidationId.indexModeChange, + "reindex please", services, "music"), new VespaRestartAction(ClusterSpec.Id.from("test"), "change", services))); DeployTester tester = createTester(hosts, modelFactories, prodZone, clock); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 7612aa5e01f..a34de472d1e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -225,25 +225,25 @@ public class ApplicationHandlerTest { clock.advance(Duration.ofSeconds(1)); expected = expected.withReady(clock.instant()); - reindex(applicationId, "?cluster="); + reindex(applicationId, "?clusterId="); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); expected = expected.withReady(clock.instant()); - reindex(applicationId, "?type=moo"); + reindex(applicationId, "?documentType=moo"); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - reindex(applicationId, "?cluster=foo,boo"); + reindex(applicationId, "?clusterId=foo,boo"); expected = expected.withReady("foo", clock.instant()) .withReady("boo", clock.instant()); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - reindex(applicationId, "?cluster=foo,boo&type=bar,baz"); + reindex(applicationId, "?clusterId=foo,boo&documentType=bar,baz"); expected = expected.withReady("foo", "bar", clock.instant()) .withReady("foo", "baz", clock.instant()) .withReady("boo", "bar", clock.instant()) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 325db1feba6..6729be20305 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; @@ -108,6 +109,17 @@ public class LbServicesProducerTest { } } + @Test + public void use_power_of_two_lb_is_configured_from_feature_flag() throws IOException, SAXException { + RegionName regionName = RegionName.from("us-east-1"); + + LbServicesConfig conf = createModelAndGetLbServicesConfig(regionName); + assertFalse(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").usePowerOfTwoChoicesLb()); + + flagSource.withBooleanFlag(Flags.USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING.id(), true); + conf = createModelAndGetLbServicesConfig(regionName); + assertTrue(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").usePowerOfTwoChoicesLb()); + } private LbServicesConfig createModelAndGetLbServicesConfig(RegionName regionName) throws IOException, SAXException { Zone zone = new Zone(Environment.prod, regionName); Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder() diff --git a/container-core/src/main/java/com/yahoo/container/Container.java b/container-core/src/main/java/com/yahoo/container/Container.java index 031d4a26d05..70167453e21 100755 --- a/container-core/src/main/java/com/yahoo/container/Container.java +++ b/container-core/src/main/java/com/yahoo/container/Container.java @@ -32,7 +32,7 @@ public class Container { private volatile FileAcquirer fileAcquirer; private volatile UrlDownloader urlDownloader; - private static Logger logger = Logger.getLogger(Container.class.getName()); + private static final Logger logger = Logger.getLogger(Container.class.getName()); // TODO: Make this final again. private static Container instance = new Container(); @@ -52,7 +52,7 @@ public class Container { } /** - * Hack. For internal use only, will be removed later + * Hack. For internal use only, will be removed later. * * Used by Application to be able to repeatedly set up containers. */ @@ -93,7 +93,7 @@ public class Container { this.componentRegistry = registry; } - //Only intended for use by the Server instance. + // Only intended for use by the Server instance. public void setupFileAcquirer(QrConfig.Filedistributor filedistributorConfig) { if (usingCustomFileAcquirer) return; @@ -109,9 +109,7 @@ public class Container { setPathAcquirer(fileAcquirer); } - /** - * Only for internal use. - */ + /** Only for internal use. */ public void setCustomFileAcquirer(FileAcquirer fileAcquirer) { if (this.fileAcquirer != null) { throw new RuntimeException("Can't change file acquirer. Is " + diff --git a/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java index 6ecb6c75f90..5bb506fbd6a 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/ApplicationBundleLoader.java @@ -21,9 +21,11 @@ import java.util.stream.Collectors; * @author Tony Vaagenes */ public class ApplicationBundleLoader { + private static final Logger log = Logger.getLogger(ApplicationBundleLoader.class.getName()); - /* Map of file refs of active bundles (not scheduled for uninstall) to the installed bundle. + /** + * Map of file refs of active bundles (not scheduled for uninstall) to the installed bundle. * * Used to: * 1. Avoid installing already installed bundles. Just an optimization, installing the same bundle location is a NOP diff --git a/container-core/src/main/java/com/yahoo/container/core/config/FileAcquirerBundleInstaller.java b/container-core/src/main/java/com/yahoo/container/core/config/FileAcquirerBundleInstaller.java index 51d77462652..97a11ce7507 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/FileAcquirerBundleInstaller.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/FileAcquirerBundleInstaller.java @@ -18,7 +18,8 @@ import java.util.logging.Logger; * @author gjoranv */ public class FileAcquirerBundleInstaller { - private static Logger log = Logger.getLogger(FileAcquirerBundleInstaller.class.getName()); + + private static final Logger log = Logger.getLogger(FileAcquirerBundleInstaller.class.getName()); private final FileAcquirer fileAcquirer; @@ -39,8 +40,8 @@ public class FileAcquirerBundleInstaller { retries++; } if (notReadable(file)) { - com.yahoo.protect.Process.logAndDie("Shutting down - unable to read bundle file with reference '" + reference - + "' and path " + file.getAbsolutePath()); + com.yahoo.protect.Process.logAndDie("Shutting down - unable to read bundle file with reference '" + + reference + "' and path " + file.getAbsolutePath()); } } diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index 9c0951c1c95..af163e88fee 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -37,7 +37,6 @@ import java.util.logging.Logger; import static com.yahoo.collections.CollectionUtil.first; - /** * For internal use only. * diff --git a/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java index 0ab89e223f6..3951e656736 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java @@ -20,6 +20,7 @@ import java.util.logging.Logger; * @author gjoranv */ public class PlatformBundleLoader { + private static final Logger log = Logger.getLogger(PlatformBundleLoader.class.getName()); private final Osgi osgi; diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java index 2ae33347408..d98a865e1fb 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java @@ -35,12 +35,13 @@ import java.util.concurrent.Executors; * */ public class HandlersConfigurerTestWrapper { - private ConfigSourceSet configSources = + + private final ConfigSourceSet configSources = new ConfigSourceSet(this.getClass().getSimpleName() + ": " + new Random().nextLong()); - private HandlersConfigurerDi configurer; + private final HandlersConfigurerDi configurer; // TODO: Remove once tests use ConfigSet rather than dir: - private final static String testFiles[] = { + private final static String[] testFiles = { "components.cfg", "handlers.cfg", "platform-bundles.cfg", @@ -143,8 +144,11 @@ public class HandlersConfigurerTestWrapper { } private static class SimpleContainerThreadpool implements ContainerThreadPool { + private final Executor executor = Executors.newCachedThreadPool(); + @Override public Executor executor() { return executor; } + } } diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java index 98c927b8efd..356f302f7c0 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/MockOsgiWrapper.java @@ -37,4 +37,5 @@ public class MockOsgiWrapper implements OsgiWrapper { @Override public void allowDuplicateBundles(Collection<Bundle> bundles) { } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java index 2ec5d34a0a6..7593e4dfecb 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -92,7 +92,7 @@ public class Coverage { } /** - * The number of documents searched for this result. If the final result + * Returns the number of documents searched for this result. If the final result * set is produced through several queries, this number will be the sum * for all the queries. */ @@ -100,18 +100,12 @@ public class Coverage { return docs; } - /** - * Total number of documents that could be searched. - * - * @return Total number of active documents - */ + /** Returns the total number of documents that could be searched. */ public long getActive() { return active; } /** - * Total number of documents that will be searchable once redistribution has settled. - * Still in beta, sematics not finalized yet. - * - * @return Total number of documents that will soon be available. + * Returns the total number of documents that will be searchable once redistribution has settled. + * Still in beta, semantics not finalized yet. */ @Beta public long getSoonActive() { return soonActive; } @@ -122,9 +116,7 @@ public class Coverage { public boolean isDegradedByAdapativeTimeout() { return (degradedReason & DEGRADED_BY_ADAPTIVE_TIMEOUT) != 0; } public boolean isDegradedByNonIdealState() { return (degradedReason == 0) && (getResultPercentage() != 100);} - /** - * @return whether the search had full coverage or not - */ + /** Returns whether the search had full coverage or not */ public boolean getFull() { switch (fullReason) { case EXPLICITLY_FULL: @@ -138,16 +130,12 @@ public class Coverage { } } - /** - * @return the number of search instances which participated successfully in the search. - */ + /** Returns the number of search instances which participated successfully in the search. */ public int getNodes() { return nodes; } - /** - * @return the number of search instances which tried to participate in the search. - */ + /** Returns the number of search instances which tried to participate in the search. */ public int getNodesTried() { return nodesTried; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/Prefix.java b/container-core/src/main/java/com/yahoo/container/handler/Prefix.java index 076e0b32a58..2fec5f07736 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Prefix.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Prefix.java @@ -49,4 +49,5 @@ public final class Prefix implements Comparable<Prefix> { public String toString() { return prefix + ": " + handler; } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java index ae313b1b04c..a07898cb1d1 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java @@ -58,7 +58,7 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex public Executor get() { return threadpool.executor(); } /** - * Shutdown the thread pool, give a grace period of 1 second before forcibly + * Shut down the thread pool, give a grace period of 1 second before forcibly * shutting down all worker threads. */ @Override diff --git a/container-core/src/main/java/com/yahoo/container/handler/Timing.java b/container-core/src/main/java/com/yahoo/container/handler/Timing.java index e52928404be..0026854ce61 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Timing.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Timing.java @@ -70,4 +70,5 @@ public class Timing { public long getTimeout() { return timeout; } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java index f2df5ddb7c5..04d03a44fb6 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java @@ -70,8 +70,7 @@ public final class VipStatusHandler extends ThreadedHttpRequestHandler { stream.write(data); } else { - throw new IllegalStateException( - "Neither file nor hardcoded data. This is a bug, please notify the Vespa team."); + throw new IllegalStateException("Neither file nor hardcoded data. This is a bug."); } stream.close(); } @@ -158,7 +157,6 @@ public final class VipStatusHandler extends ThreadedHttpRequestHandler { * out of capacity. This is the default behavior. */ @Inject - @SuppressWarnings("unused") // injected public VipStatusHandler(VipStatusConfig vipConfig, Metric metric, VipStatus vipStatus) { // One thread should be enough for status handling - otherwise something else is completely wrong, // in which case this will eventually start returning a 503 (due to work rejection) as the bounded diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/ErrorResponse.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/ErrorResponse.java index 1fcde746878..c59dc2939a5 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/ErrorResponse.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/ErrorResponse.java @@ -13,9 +13,10 @@ import static java.util.logging.Level.WARNING; * @author gjoranv */ public class ErrorResponse extends JsonResponse { - private static Logger log = Logger.getLogger(ErrorResponse.class.getName()); - private static ObjectMapper objectMapper = new ObjectMapper(); + private static final Logger log = Logger.getLogger(ErrorResponse.class.getName()); + + private static final ObjectMapper objectMapper = new ObjectMapper(); public ErrorResponse(int code, String message) { super(code, asErrorJson(message != null ? message : "<null>")); @@ -29,4 +30,5 @@ public class ErrorResponse extends JsonResponse { return "Could not encode error message to json, check the log for details."; } } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/JsonResponse.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/JsonResponse.java index def06ce9de3..fc817332079 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/JsonResponse.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/JsonResponse.java @@ -11,6 +11,7 @@ import java.nio.charset.Charset; * @author gjoranv */ public class JsonResponse extends HttpResponse { + private final byte[] data; public JsonResponse(int code, String data) { @@ -27,4 +28,5 @@ public class JsonResponse extends HttpResponse { public void render(OutputStream outputStream) throws IOException { outputStream.write(data); } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java index 78ea62e1b3a..a4a092b02ad 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java @@ -74,4 +74,5 @@ public class MetricsV2Handler extends HttpHandlerBase { static String consumerQuery(String consumer) { return (consumer == null || consumer.isEmpty()) ? "" : "?consumer=" + consumer; } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java index e33f2f47828..00fb488489e 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java @@ -20,6 +20,9 @@ import org.apache.http.impl.client.CloseableHttpClient; import static com.yahoo.container.handler.metrics.MetricsV2Handler.consumerQuery; import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +/** + * @author Oracien + */ public class PrometheusV1Handler extends HttpHandlerBase{ public static final String V1_PATH = "/prometheus/v1"; diff --git a/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java b/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java index eef80e95b3d..0228bc06c51 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java +++ b/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java @@ -43,18 +43,18 @@ import java.util.logging.Logger; @Beta public class MockService extends LoggingRequestHandler { - private MockServiceHandler handler; + private final MockServiceHandler handler; /** * Create a mock service that mocks an external service using data provided via file distribution. * A custom handler can be created by subclassing and overriding the createHandler method. * - * @param executor An {@link Executor} used to create threads. - * @param accessLog An {@link AccessLog} where requests will be logged. - * @param fileAcquirer A {@link FileAcquirer} which is used to fetch file from config. - * @param config A {@link MockserviceConfig} for this service. - * @throws InterruptedException if unable to get data file within timeout. - * @throws IOException if unable to create handler due to some IO errors. + * @param executor used to create threads + * @param accessLog where requests will be logged + * @param fileAcquirer used to fetch file from config + * @param config the mock config for this service + * @throws InterruptedException if unable to get data file within timeout + * @throws IOException if unable to create handler due to some IO errors */ public MockService(Executor executor, AccessLog accessLog, FileAcquirer fileAcquirer, MockserviceConfig config, Metric metric) throws InterruptedException, IOException { super(executor, accessLog, metric); @@ -65,9 +65,9 @@ public class MockService extends LoggingRequestHandler { /** * Create a handler for a file. Override this method to handle a custom file syntax of your own. * - * @param dataFile A file to read. - * @return a {@link MockServiceHandler} used to handle requests. - * @throws IOException if errors occured when loading the file + * @param dataFile the file to read + * @return the handler used to handle requests + * @throws IOException if errors occurred when loading the file */ protected MockServiceHandler createHandler(File dataFile) throws IOException { if (!dataFile.getName().endsWith(".txt")) { @@ -210,7 +210,7 @@ public class MockService extends LoggingRequestHandler { } } - private class ExceptionResponse extends HttpResponse { + private static class ExceptionResponse extends HttpResponse { private final Exception e; public ExceptionResponse(int code, Exception e) { super(code); @@ -224,4 +224,5 @@ public class MockService extends LoggingRequestHandler { } } } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/test/MockServiceHandler.java b/container-core/src/main/java/com/yahoo/container/handler/test/MockServiceHandler.java index 0a246431e43..2ef3d66d501 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/test/MockServiceHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/test/MockServiceHandler.java @@ -8,17 +8,17 @@ import com.yahoo.container.jdisc.HttpRequest; * A service handler that is able to map a request to a key and retrieve a value given a key. * * @author Ulf Lilleengen - * @since 5.1.21 */ @Beta public interface MockServiceHandler { + /** * Create a custom Key given a http request. This will be called for each request, and allows a handler * to customize its key format. * @param request The client http request. * @return a {@link Key} used to query for the value. */ - public Key createKey(HttpRequest request); + Key createKey(HttpRequest request); /** * Lookup a {@link Value} for a {@link Key}. Returns null if the key is not found. @@ -26,9 +26,10 @@ public interface MockServiceHandler { * @param key The {@link Key} to look up. * @return A {@link Value} used as response. */ - public Value get(Key key); + Value get(Key key); + + final class Value { - public final class Value { public final int returnCode; public final byte[] data; public final String contentType; @@ -38,10 +39,13 @@ public interface MockServiceHandler { this.data = data; this.contentType = contentType; } + } - public interface Key { - public int hashCode(); - public boolean equals(Object other); + interface Key { + + int hashCode(); + boolean equals(Object other); + } } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java index 46b3a86798b..6bed4a6f442 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/DefaultContainerThreadpool.java @@ -50,7 +50,9 @@ public class DefaultContainerThreadpool extends AbstractComponent implements Aut } @Override public Executor executor() { return threadpool; } + @Override public void close() { closeInternal(); } + @Override public void deconstruct() { closeInternal(); super.deconstruct(); } /** diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java index 771c1da82b6..8e0d11c3171 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ExecutorServiceWrapper.java @@ -28,10 +28,12 @@ class ExecutorServiceWrapper extends ForwardingExecutorService { private final Thread metricReporter; private final AtomicBoolean closed = new AtomicBoolean(false); - ExecutorServiceWrapper( - WorkerCompletionTimingThreadPoolExecutor wrapped, - ThreadPoolMetric metric, ProcessTerminator processTerminator, - long maxThreadExecutionTimeMillis, String name, int queueCapacity) { + ExecutorServiceWrapper(WorkerCompletionTimingThreadPoolExecutor wrapped, + ThreadPoolMetric metric, + ProcessTerminator processTerminator, + long maxThreadExecutionTimeMillis, + String name, + int queueCapacity) { this.wrapped = wrapped; this.metric = metric; this.processTerminator = processTerminator; diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java index 56f8319c110..1f64cdbdc40 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java @@ -17,21 +17,18 @@ import java.util.concurrent.atomic.AtomicLong; */ class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor { - - volatile long lastThreadAssignmentTimeMillis = System.currentTimeMillis(); private final AtomicLong startedCount = new AtomicLong(0); private final AtomicLong completedCount = new AtomicLong(0); private final ThreadPoolMetric metric; - WorkerCompletionTimingThreadPoolExecutor( - int corePoolSize, - int maximumPoolSize, - long keepAliveTime, - TimeUnit unit, - BlockingQueue<Runnable> workQueue, - ThreadFactory threadFactory, - ThreadPoolMetric metric) { + WorkerCompletionTimingThreadPoolExecutor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue<Runnable> workQueue, + ThreadFactory threadFactory, + ThreadPoolMetric metric) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); this.metric = metric; } @@ -56,5 +53,6 @@ class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor { public int getActiveCount() { return (int)(startedCount.get() - completedCount.get()); } + } diff --git a/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java b/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java index 31bceca9337..688d2c1c5be 100644 --- a/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java +++ b/container-core/src/main/java/com/yahoo/container/http/filter/FilterChainRepository.java @@ -38,6 +38,7 @@ import static java.util.stream.Collectors.toSet; * @author bjorncs */ public class FilterChainRepository extends AbstractComponent { + private static final Logger log = Logger.getLogger(FilterChainRepository.class.getName()); private final ComponentRegistry<Object> filterAndChains; @@ -198,4 +199,5 @@ public class FilterChainRepository extends AbstractComponent { throw new IllegalArgumentException("Unsupported filter type: " + filter.getClass().getName()); } } + } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/AsyncHttpResponse.java b/container-core/src/main/java/com/yahoo/container/jdisc/AsyncHttpResponse.java index 592cbff8440..6ed467a465c 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/AsyncHttpResponse.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/AsyncHttpResponse.java @@ -32,13 +32,9 @@ public abstract class AsyncHttpResponse extends HttpResponse { * output (using the provided channel and completion handler) when (async) * rendering is completed. * - * @param output - * the stream to which content should be rendered - * @param networkChannel - * the channel which must be closed on completion - * @param handler - * the completion handler to submit when closing the channel, may - * be null + * @param output the stream to which content should be rendered + * @param networkChannel the channel which must be closed on completion + * @param handler the completion handler to submit when closing the channel, may be null */ public abstract void render(OutputStream output, ContentChannel networkChannel, CompletionHandler handler) throws IOException; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java b/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java index 329889e70c0..1d4c20efe5e 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java @@ -29,7 +29,7 @@ public class ContentChannelOutputStream extends OutputStream implements Writable private boolean failed = false; private final Object failLock = new Object(); - public ContentChannelOutputStream(final ContentChannel endpoint) { + public ContentChannelOutputStream(ContentChannel endpoint) { this.endpoint = endpoint; buffer = new BufferChain(this); } @@ -38,7 +38,7 @@ public class ContentChannelOutputStream extends OutputStream implements Writable * Buffered write of a single byte. */ @Override - public void write(final int b) throws IOException { + public void write(int b) throws IOException { try { buffer.append((byte) b); } catch (RuntimeException e) { @@ -74,8 +74,7 @@ public class ContentChannelOutputStream extends OutputStream implements Writable * It is in other words safe to recycle the array {@code b}. */ @Override - public void write(final byte[] b, final int off, final int len) - throws IOException { + public void write(byte[] b, int off, int len) throws IOException { nonCopyingWrite(Arrays.copyOfRange(b, off, off + len)); } @@ -85,7 +84,7 @@ public class ContentChannelOutputStream extends OutputStream implements Writable * It is in other words safe to recycle the array {@code b}. */ @Override - public void write(final byte[] b) throws IOException { + public void write(byte[] b) throws IOException { nonCopyingWrite(Arrays.copyOf(b, b.length)); } @@ -94,8 +93,7 @@ public class ContentChannelOutputStream extends OutputStream implements Writable * <i>transferring</i> ownership of that array to this stream. It is in * other words <i>not</i> safe to recycle the array {@code b}. */ - public void nonCopyingWrite(final byte[] b, final int off, final int len) - throws IOException { + public void nonCopyingWrite(byte[] b, int off, int len) throws IOException { try { buffer.append(b, off, len); } catch (RuntimeException e) { @@ -108,7 +106,7 @@ public class ContentChannelOutputStream extends OutputStream implements Writable * <i>transferring</i> ownership of that array to this stream. It is in * other words <i>not</i> safe to recycle the array {@code b}. */ - public void nonCopyingWrite(final byte[] b) throws IOException { + public void nonCopyingWrite(byte[] b) throws IOException { try { buffer.append(b); } catch (RuntimeException e) { @@ -125,27 +123,23 @@ public class ContentChannelOutputStream extends OutputStream implements Writable * the ByteBuffer to this stream. */ @Override - public void send(final ByteBuffer src) throws IOException { - // Don't do a buffer.flush() from here, this method is used by the - // buffer itself + public void send(ByteBuffer src) throws IOException { + // Don't do a buffer.flush() from here, this method is used by the buffer itself try { - byteBufferData += (long) src.remaining(); + byteBufferData += src.remaining(); endpoint.write(src, new LoggingCompletionHandler()); } catch (RuntimeException e) { throw new IOException(Exceptions.toMessageString(e), e); } } - /** - * Give the number of bytes written. - * - * @return the number of bytes written to this stream - */ + /** Returns the number of bytes written to this stream */ public long written() { return buffer.appended() + byteBufferData; } class LoggingCompletionHandler implements CompletionHandler { + @Override public void completed() { } @@ -166,4 +160,5 @@ public class ContentChannelOutputStream extends OutputStream implements Writable } } } + } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/EmptyResponse.java b/container-core/src/main/java/com/yahoo/container/jdisc/EmptyResponse.java index 88c6291fb26..737017b7957 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/EmptyResponse.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/EmptyResponse.java @@ -5,8 +5,7 @@ import java.io.IOException; import java.io.OutputStream; /** - * Placeholder response when no content, only headers and status is to be - * returned. + * Placeholder response when no content, only headers and status is to be returned. * * @author Steinar Knutsen */ @@ -19,4 +18,5 @@ public class EmptyResponse extends HttpResponse { public void render(OutputStream outputStream) throws IOException { // NOP } + } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java index edd24fed515..e202442479f 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequest.java @@ -55,7 +55,7 @@ public class HttpRequest { InputStream requestData = null; URI uri = null; CurrentContainer container = null; - private String nag = " must be set before the attempted operation."; + private final String nag = " must be set before the attempted operation."; SocketAddress remoteAddress; private void boom(Object ref, String what) { @@ -99,9 +99,7 @@ public class HttpRequest { * {@link #jdiscRequest(com.yahoo.jdisc.http.HttpRequest)} before * instantiating any HTTP request. * - * @param request - * source for defaults and parent JDisc request, may be null - * + * @param request source for defaults and parent JDisc request, may be null * @see HttpRequest#createTestRequest(String, com.yahoo.jdisc.http.HttpRequest.Method) */ public Builder(HttpRequest request) { @@ -111,9 +109,7 @@ public class HttpRequest { /** * Instantiate a request builder with defaults from an existing request. * - * @param request - * parent JDisc request - * + * @param request parent JDisc request * @see HttpRequest#createTestRequest(String, com.yahoo.jdisc.http.HttpRequest.Method) */ public Builder(com.yahoo.jdisc.http.HttpRequest request) { @@ -216,8 +212,7 @@ public class HttpRequest { } /** - * Start of API for synchronous HTTP request dispatch. Not yet ready for - * use. + * Start of API for synchronous HTTP request dispatch. Not yet ready for use. * * @return a new client request */ @@ -244,8 +239,7 @@ public class HttpRequest { } /** - * Start of API for synchronous HTTP request dispatch. Not yet ready for - * use. + * Start of API for synchronous HTTP request dispatch. Not yet ready for use. * * @return a new server request */ @@ -277,8 +271,7 @@ public class HttpRequest { return new HttpRequest(serverRequest, requestData, properties); } - private void setParameters( - com.yahoo.jdisc.http.HttpRequest request) { + private void setParameters(com.yahoo.jdisc.http.HttpRequest request) { for (Map.Entry<String, String> entry : properties.entrySet()) { request.parameters().put(entry.getKey(), wrap(entry.getValue())); } @@ -290,10 +283,8 @@ public class HttpRequest { * Wrap a JDisc HTTP request in a synchronous API. The properties from the * JDisc request will be copied into the HTTP request. * - * @param jdiscHttpRequest - * the JDisc request - * @param requestData - * the associated input stream, e.g. with POST request + * @param jdiscHttpRequest the JDisc request + * @param requestData the associated input stream, e.g. with POST request */ public HttpRequest(com.yahoo.jdisc.http.HttpRequest jdiscHttpRequest, InputStream requestData) { this(jdiscHttpRequest, requestData, null); @@ -308,13 +299,10 @@ public class HttpRequest { * will obviously not be reflected by the request. The same applies for * JDisc parameters. * - * @param jdiscHttpRequest - * the JDisc request - * @param requestData - * the associated input stream, e.g. with POST request - * @param propertyOverrides - * properties which should not have the same settings as in the - * parent JDisc request, may be null + * @param jdiscHttpRequest the JDisc request + * @param requestData the associated input stream, e.g. with POST request + * @param propertyOverrides properties which should not have the same settings as in the + * parent JDisc request, may be null */ public HttpRequest(com.yahoo.jdisc.http.HttpRequest jdiscHttpRequest, InputStream requestData, Map<String, String> propertyOverrides) { @@ -495,8 +483,7 @@ public class HttpRequest { * Helper method to parse boolean request flags, using * Boolean.parseBoolean(String). Unset values are regarded as false. * - * @param name - * the name of a request property + * @param name the name of a request property * @return whether the property has been explicitly set to true */ public boolean getBooleanProperty(String name) { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index 19460053469..064b6cf6279 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -37,24 +37,30 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { } public static class Context { + final Executor executor; final AccessLog accessLog; final Metric metric; + @Inject public Context(Executor executor, AccessLog accessLog, Metric metric) { this.executor = executor; this.accessLog = accessLog; this.metric = metric; } + public Context(Context other) { this.executor = other.executor; this.accessLog = other.accessLog; this.metric = other.metric; } + public Executor getExecutor() { return executor; } public AccessLog getAccessLog() { return accessLog; } public Metric getMetric() { return metric; } + } + public static Context testOnlyContext() { return new Context(new Executor() { @Override @@ -254,15 +260,14 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { } else { // Not running on JDisc http layer (Jetty), e.g unit tests AccessLogEntry accessLogEntry = new AccessLogEntry(); - populateAccessLogEntryNotCreatedByHttpServer( - accessLogEntry, - jdiscRequest, - extendedResponse.getTiming(), - httpRequest.getUri().toString(), - commitStartTime, - startTime, - rendererWiring.written(), - httpResponse.getStatus()); + populateAccessLogEntryNotCreatedByHttpServer(accessLogEntry, + jdiscRequest, + extendedResponse.getTiming(), + httpRequest.getUri().toString(), + commitStartTime, + startTime, + rendererWiring.written(), + httpResponse.getStatus()); accessLog.log(accessLogEntry); entry = accessLogEntry; } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java index a3e264c16ee..faa30bd109d 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java @@ -28,7 +28,7 @@ import java.util.concurrent.TimeUnit; @Beta public class RequestHandlerTestDriver implements AutoCloseable { - private TestDriver driver; + private final TestDriver driver; private MockResponseHandler responseHandler = null; @@ -152,7 +152,7 @@ public class RequestHandlerTestDriver implements AutoCloseable { StringBuilder b = new StringBuilder(); while (content.available()>0) { ByteBuffer nextBuffer = content.read(); - b.append(Charset.forName("utf-8").decode(nextBuffer).toString()); + b.append(Charset.forName("utf-8").decode(nextBuffer)); } return b.toString(); } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index c46488694de..9687697d6f6 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -69,9 +69,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler @Override public final void handleRequest(Request request, BufferedContentChannel requestContent, ResponseHandler responseHandler) { - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "In " + this.getClass() + ".handleRequest()"); - } + log.log(Level.FINE, () -> "In " + this.getClass() + ".handleRequest()"); com.yahoo.jdisc.http.HttpRequest jdiscRequest = asHttpRequest(request); HttpRequest httpRequest = new HttpRequest(jdiscRequest, new UnsafeContentInputStream(requestContent.toReadable())); LazyContentChannel channel = null; @@ -95,8 +93,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler } /** Render and return whether the channel was closed */ - private void render(HttpRequest request, HttpResponse httpResponse, - LazyContentChannel channel, long startTime) { + private void render(HttpRequest request, HttpResponse httpResponse, LazyContentChannel channel, long startTime) { LoggingCompletionHandler logOnCompletion = null; ContentChannelOutputStream output = null; try { @@ -139,7 +136,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler private boolean closed = false; // Fields needed to lazily create or close the channel */ - private HttpRequest httpRequest; + private final HttpRequest httpRequest; private HttpResponse httpResponse; private final ResponseHandler responseHandler; private final Metric metric; @@ -227,29 +224,27 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler /** * Override this to implement custom access logging. * - * @param startTime - * execution start - * @param renderStartTime - * start of output rendering - * @param response - * the response which the log entry regards - * @param httpRequest - * the incoming HTTP request - * @param rendererWiring - * the stream the rendered response is written to, used for - * fetching length of rendered response + * @param startTime execution start + * @param renderStartTime start of output rendering + * @param response the response which the log entry regards + * @param httpRequest the incoming HTTP request + * @param rendererWiring the stream the rendered response is written to, used for + * fetching length of rendered response */ - protected LoggingCompletionHandler createLoggingCompletionHandler( - long startTime, long renderStartTime, HttpResponse response, - HttpRequest httpRequest, ContentChannelOutputStream rendererWiring) { + protected LoggingCompletionHandler createLoggingCompletionHandler(long startTime, + long renderStartTime, + HttpResponse response, + HttpRequest httpRequest, + ContentChannelOutputStream rendererWiring) { return null; } protected com.yahoo.jdisc.http.HttpRequest asHttpRequest(Request request) { if (!(request instanceof com.yahoo.jdisc.http.HttpRequest)) { - throw new IllegalArgumentException("Expected " - + com.yahoo.jdisc.http.HttpRequest.class.getName() + ", got " + request.getClass().getName()); + throw new IllegalArgumentException("Expected " + com.yahoo.jdisc.http.HttpRequest.class.getName() + + ", got " + request.getClass().getName()); } return (com.yahoo.jdisc.http.HttpRequest) request; } + } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index 4c93613603b..446ee90c205 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -269,6 +269,7 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { private static class NullFeedContext implements Context { private static final NullFeedContext INSTANCE = new NullFeedContext(); } + } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java b/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java index f6a089d6fd2..3236f7d2407 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java @@ -47,11 +47,7 @@ public final class VespaHeaders { private static final Tuple2<Boolean, Integer> NO_MATCH = new Tuple2<>(false, Response.Status.OK); public static boolean benchmarkCoverage(boolean benchmarkOutput, HeaderFields headers) { - if (benchmarkOutput && headers.get(BenchmarkingHeaders.REQUEST_COVERAGE) != null) { - return true; - } else { - return false; - } + return benchmarkOutput && headers.get(BenchmarkingHeaders.REQUEST_COVERAGE) != null; } /** Returns true if this is a benchmarking request, according to headers */ @@ -60,14 +56,14 @@ public final class VespaHeaders { } /** - * Add search benchmark output to the HTTP getHeaders + * Add search benchmark output to the HTTP getHeaders. * - * @param responseHeaders The response to write the headers to. - * @param benchmarkCoverage True to include coverage headers. - * @param t The Timing to read data from. - * @param c The Counts to read data from. - * @param errorCount The error count. - * @param coverage The Coverage to read data from. + * @param responseHeaders the response to write the headers to + * @param benchmarkCoverage true to include coverage headers + * @param t the Timing to read data from + * @param c the Counts to read data from + * @param errorCount the error count + * @param coverage the Coverage to read data from */ public static void benchmarkOutput(HeaderFields responseHeaders, boolean benchmarkCoverage, Timing t, HitCounts c, int errorCount, Coverage coverage) { @@ -106,10 +102,10 @@ public final class VespaHeaders { /** * (during normal execution) return 200 unless this is not a success or a 4xx error is requested. * - * @param isSuccess Whether or not the response represents a success. - * @param mainError The main error of the response, if any. - * @param allErrors All the errors of the response, if any. - * @return The status code of the given response. + * @param isSuccess whether or not the response represents a success + * @param mainError the main error of the response, if any + * @param allErrors all the errors of the response, if any + * @return the status code of the given response */ public static int getStatus(boolean isSuccess, ErrorMessage mainError, Iterator<? extends ErrorMessage> allErrors) { // Do note, SearchResponse has its own implementation of isSuccess() @@ -129,7 +125,7 @@ public final class VespaHeaders { Iterator<? extends ErrorMessage> errorIterator = allErrors; if (errorIterator != null && errorIterator.hasNext()) { - for (; errorIterator.hasNext();) { + while (errorIterator.hasNext()) { ErrorMessage error = errorIterator.next(); Tuple2<Boolean, Integer> status = chooseWebServiceStatus(error); if (status.first) { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/FileWrapper.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/FileWrapper.java index 6e22e02eb5b..3e127b87017 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/FileWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/FileWrapper.java @@ -24,4 +24,5 @@ public class FileWrapper { boolean isRegularFile(Path path) { return Files.isRegularFile(path); } + } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/GaugeMetric.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/GaugeMetric.java index 9b89b8abe52..9a195710c8f 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/GaugeMetric.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/GaugeMetric.java @@ -20,7 +20,7 @@ public final class GaugeMetric extends MetricValue { private double min; private double sum; private long count; - private Optional<List<Tuple2<String, Double>>> percentiles; + private final Optional<List<Tuple2<String, Double>>> percentiles; private GaugeMetric(double last, double max, double min, double sum, long count, Optional<List<Tuple2<String, Double>>> percentiles) { this.last = last; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/HostLifeGatherer.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/HostLifeGatherer.java index 080a5a8dc32..730f7bc13cd 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/HostLifeGatherer.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/HostLifeGatherer.java @@ -44,4 +44,5 @@ public class HostLifeGatherer { return jsonObject; } + } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/JSONObjectWithLegibleException.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/JSONObjectWithLegibleException.java index dc1bfb89197..d22dd9d6f4b 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/JSONObjectWithLegibleException.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/JSONObjectWithLegibleException.java @@ -13,6 +13,7 @@ import java.util.Map; * @author gjoranv */ class JSONObjectWithLegibleException extends JSONObject { + @Override public JSONObject put(String s, boolean b) { try { @@ -80,7 +81,7 @@ class JSONObjectWithLegibleException extends JSONObject { private String getErrorMessage(String key, Object value, JSONException e) { return "Trying to add invalid JSON object with key '" + key + - "' and value '" + value + - "' - " + e.getMessage(); + "' and value '" + value + "' - " + e.getMessage(); } + } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricGatherer.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricGatherer.java index 061ce7138ad..6a06a6362f5 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricGatherer.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricGatherer.java @@ -7,8 +7,9 @@ import java.util.ArrayList; import java.util.List; /** + * Gathers metrics regarding currently processing coredumps and host life. + * * @author olaa - * Gathers metrics regarding currently processing coredumps and host life */ public class MetricGatherer { @@ -17,7 +18,8 @@ public class MetricGatherer { List<JSONObject> packetList = new ArrayList<>(); packetList.add(CoredumpGatherer.gatherCoredumpMetrics(fileWrapper)); if (System.getProperty("os.name").contains("nux")) - packetList.add(HostLifeGatherer.getHostLifePacket(fileWrapper)); + packetList.add(HostLifeGatherer.getHostLifePacket(fileWrapper)); return packetList; } + } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java index d1036db5c6f..c1a6f650a9c 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java @@ -43,6 +43,7 @@ import static com.yahoo.container.jdisc.state.StateHandler.getSnapshotPreprocess * @author gjoranv */ public class MetricsPacketsHandler extends AbstractRequestHandler { + static final String APPLICATION_KEY = "application"; static final String TIMESTAMP_KEY = "timestamp"; static final String STATUS_CODE_KEY = "status_code"; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/SnapshotProvider.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/SnapshotProvider.java index 4967fd1f162..d693bf97bd8 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/SnapshotProvider.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/SnapshotProvider.java @@ -7,7 +7,7 @@ import java.io.PrintStream; * An interface for components supplying a state snapshot where persistence and * other pre-processing has been done. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public interface SnapshotProvider { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index ccd0864b3ab..40a0ef10fbc 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -136,4 +136,5 @@ public class StateMonitor extends AbstractComponent { } }); } + } diff --git a/container-core/src/main/java/com/yahoo/container/servlet/ServletProvider.java b/container-core/src/main/java/com/yahoo/container/servlet/ServletProvider.java index 903c01a27f7..213063a725d 100644 --- a/container-core/src/main/java/com/yahoo/container/servlet/ServletProvider.java +++ b/container-core/src/main/java/com/yahoo/container/servlet/ServletProvider.java @@ -11,14 +11,11 @@ import org.eclipse.jetty.servlet.ServletHolder; */ public class ServletProvider implements Provider<ServletHolder> { - private ServletHolder servletHolder; + private final ServletHolder servletHolder; public ServletProvider(Servlet servlet, ServletConfigConfig servletConfigConfig) { servletHolder = new ServletHolder(servlet); - - servletConfigConfig.map().forEach( (key, value) -> - servletHolder.setInitParameter(key, value) - ); + servletConfigConfig.map().forEach( (key, value) -> servletHolder.setInitParameter(key, value)); } @Override diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java index ffce3649419..d49c548d25c 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java @@ -10,8 +10,9 @@ import javax.xml.datatype.DatatypeFactory; * @author Einar M R Rosenvinge * @deprecated Do not use! */ -@Deprecated +@Deprecated // TODO: Remove on Vespa 8 public class DatatypeFactoryProvider implements Provider<DatatypeFactory> { + public static final String FACTORY_CLASS = DatatypeFactory.DATATYPEFACTORY_IMPLEMENTATION_CLASS; @Override @@ -25,4 +26,5 @@ public class DatatypeFactoryProvider implements Provider<DatatypeFactory> { @Override public void deconstruct() { } + } diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java index 37b8dff8bf4..c81d173e1ed 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java @@ -9,7 +9,7 @@ import javax.xml.parsers.DocumentBuilderFactory; * @author Einar M R Rosenvinge * @deprecated Do not use! */ -@Deprecated +@Deprecated // TODO: Remove on Vespa 8 public class DocumentBuilderFactoryProvider implements Provider<DocumentBuilderFactory> { public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"; diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java index a8ac55a8aca..0d0b79d8ce7 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java @@ -9,7 +9,7 @@ import javax.xml.parsers.SAXParserFactory; * @author Einar M R Rosenvinge * @deprecated Do not use! */ -@Deprecated +@Deprecated // TODO: Remove on Vespa 8 public class SAXParserFactoryProvider implements Provider<SAXParserFactory> { public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl"; diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java index bbcdf7c9553..0d69e129492 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java @@ -10,7 +10,7 @@ import javax.xml.validation.SchemaFactory; * @author Einar M R Rosenvinge * @deprecated Do not use! */ -@Deprecated +@Deprecated // TODO: Remove on Vespa 8 public class SchemaFactoryProvider implements Provider<SchemaFactory> { public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.validation.XMLSchemaFactory"; diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java index 974d2e6a259..071a576abe8 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java @@ -9,7 +9,7 @@ import javax.xml.transform.TransformerFactory; * @author Einar M R Rosenvinge * @deprecated Do not use! */ -@Deprecated +@Deprecated // TODO: Remove on Vespa 8 public class TransformerFactoryProvider implements Provider<TransformerFactory> { public static final String FACTORY_CLASS = "com.sun.org.apache.xalan.internal.xsltc.trax.TransformerFactoryImpl"; diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java index 702ecedcc93..f5e1e666072 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java @@ -9,7 +9,7 @@ import javax.xml.stream.XMLEventFactory; * @author Einar M R Rosenvinge * @deprecated Do not use! */ -@Deprecated +@Deprecated // TODO: Remove on Vespa 8 public class XMLEventFactoryProvider implements Provider<XMLEventFactory> { public static final String FACTORY_CLASS = "com.sun.xml.internal.stream.events.XMLEventFactoryImpl"; diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java index 9f3518525de..99eb6df7093 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java @@ -9,7 +9,7 @@ import javax.xml.stream.XMLInputFactory; * @author Einar M R Rosenvinge * @deprecated Do not use! */ -@Deprecated +@Deprecated // TODO: Remove on Vespa 8 public class XMLInputFactoryProvider implements Provider<XMLInputFactory> { private static final String INPUT_FACTORY_INTERFACE = XMLInputFactory.class.getName(); diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java index ab28ba2e923..c5a21dae410 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java @@ -9,7 +9,7 @@ import javax.xml.stream.XMLOutputFactory; * @author Einar M R Rosenvinge * @deprecated Do not use! */ -@Deprecated +@Deprecated // TODO: Remove on Vespa 8 public class XMLOutputFactoryProvider implements Provider<XMLOutputFactory> { public static final String FACTORY_CLASS = "com.sun.xml.internal.stream.XMLOutputFactoryImpl"; diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java index 407369032cf..23ef9995caf 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java @@ -10,7 +10,7 @@ import javax.xml.xpath.XPathFactoryConfigurationException; * @author Einar M R Rosenvinge * @deprecated Do not use! */ -@Deprecated +@Deprecated // TODO: Remove on Vespa 8 public class XPathFactoryProvider implements Provider<XPathFactory> { public static final String FACTORY_CLASS = "com.sun.org.apache.xpath.internal.jaxp.XPathFactoryImpl"; diff --git a/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java b/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java index ace5a7ab304..92cdfe8d918 100644 --- a/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java +++ b/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java @@ -17,8 +17,8 @@ import com.yahoo.language.opennlp.OpenNlpLinguistics; @SuppressWarnings("unused") // Injected public class DefaultLinguisticsProvider implements Provider<Linguistics> { - // Use lazy initialization to avoid expensive (memory-wise) instantiation f - private volatile Supplier<Linguistics> linguisticsSupplier = Suppliers.memoize(OpenNlpLinguistics::new); + // Use lazy initialization to avoid expensive (memory-wise) instantiation + private final Supplier<Linguistics> linguisticsSupplier = Suppliers.memoize(OpenNlpLinguistics::new); @Inject public DefaultLinguisticsProvider() { } diff --git a/container-core/src/main/java/com/yahoo/osgi/Osgi.java b/container-core/src/main/java/com/yahoo/osgi/Osgi.java index 513e7883594..54f9ad48703 100644 --- a/container-core/src/main/java/com/yahoo/osgi/Osgi.java +++ b/container-core/src/main/java/com/yahoo/osgi/Osgi.java @@ -27,4 +27,5 @@ public interface Osgi { default boolean hasFelixFramework() { return false; } + } diff --git a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java index b34442d50a9..97e2367bac2 100644 --- a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java +++ b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java @@ -19,6 +19,7 @@ import java.util.logging.Logger; * @author gjoranv */ public class OsgiImpl implements Osgi { + private static final Logger log = Logger.getLogger(OsgiImpl.class.getName()); private final OsgiFramework jdiscOsgi; diff --git a/container-core/src/main/java/com/yahoo/processing/handler/ProcessingResponse.java b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingResponse.java index 2727d111829..a8ccc28c751 100644 --- a/container-core/src/main/java/com/yahoo/processing/handler/ProcessingResponse.java +++ b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingResponse.java @@ -31,8 +31,7 @@ import com.yahoo.processing.response.DataList; * wrapper of the knowhow needed to render the Response from processing. * * @author bratseth - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @since 5.1.12 + * @author Steinar Knutsen */ public class ProcessingResponse extends AsyncHttpResponse { @@ -44,7 +43,6 @@ public class ProcessingResponse extends AsyncHttpResponse { /** True if the return status has been set explicitly and should not be further changed */ private boolean explicitStatusSet = false; - @SuppressWarnings("unchecked") public ProcessingResponse(int status, com.yahoo.processing.Request processingRequest, com.yahoo.processing.Response processingResponse, Renderer renderer, diff --git a/container-core/src/main/java/com/yahoo/processing/handler/ProcessingTestDriver.java b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingTestDriver.java index 8fc0b49c71b..d4e55dbc556 100644 --- a/container-core/src/main/java/com/yahoo/processing/handler/ProcessingTestDriver.java +++ b/container-core/src/main/java/com/yahoo/processing/handler/ProcessingTestDriver.java @@ -23,7 +23,6 @@ import java.util.concurrent.Executors; * Create an instance of this to test making processing requests and get the response or response data. * * @author bratseth - * @since 5.21 */ @Beta public class ProcessingTestDriver extends RequestHandlerTestDriver { @@ -31,20 +30,20 @@ public class ProcessingTestDriver extends RequestHandlerTestDriver { private final ProcessingHandler processingHandler; public ProcessingTestDriver(Collection<Chain<Processor>> chains) { - this(chains, new ComponentRegistry<Renderer>()); + this(chains, new ComponentRegistry<>()); } public ProcessingTestDriver(String binding, Collection<Chain<Processor>> chains) { - this(chains, new ComponentRegistry<Renderer>()); + this(chains, new ComponentRegistry<>()); } @SafeVarargs @SuppressWarnings("varargs") public ProcessingTestDriver(Chain<Processor> ... chains) { - this(Arrays.asList(chains), new ComponentRegistry<Renderer>()); + this(Arrays.asList(chains), new ComponentRegistry<>()); } @SafeVarargs @SuppressWarnings("varargs") public ProcessingTestDriver(String binding, Chain<Processor> ... chains) { - this(binding, Arrays.asList(chains), new ComponentRegistry<Renderer>()); + this(binding, Arrays.asList(chains), new ComponentRegistry<>()); } public ProcessingTestDriver(Collection<Chain<Processor>> chains, ComponentRegistry<Renderer> renderers) { this(createProcessingHandler(chains, renderers, AccessLog.voidAccessLog())); @@ -64,7 +63,7 @@ public class ProcessingTestDriver extends RequestHandlerTestDriver { public ProcessingTestDriver(Chain<Processor> chain, AccessLogInterface accessLogInterface) { this(createProcessingHandler( Collections.singleton(chain), - new ComponentRegistry<Renderer>(), + new ComponentRegistry<>(), createAccessLog(accessLogInterface))); } @@ -76,10 +75,9 @@ public class ProcessingTestDriver extends RequestHandlerTestDriver { return new AccessLog(componentRegistry); } - private static ProcessingHandler createProcessingHandler( - Collection<Chain<Processor>> chains, - ComponentRegistry<Renderer> renderers, - AccessLog accessLog) { + private static ProcessingHandler createProcessingHandler(Collection<Chain<Processor>> chains, + ComponentRegistry<Renderer> renderers, + AccessLog accessLog) { Executor executor = Executors.newSingleThreadExecutor(); ChainRegistry<Processor> registry = new ChainRegistry<>(); diff --git a/container-core/src/main/java/com/yahoo/processing/handler/ResponseHeaders.java b/container-core/src/main/java/com/yahoo/processing/handler/ResponseHeaders.java index 0267b892878..3a21c5a3ec9 100644 --- a/container-core/src/main/java/com/yahoo/processing/handler/ResponseHeaders.java +++ b/container-core/src/main/java/com/yahoo/processing/handler/ResponseHeaders.java @@ -14,7 +14,6 @@ import java.util.Map; * A Response may contain multiple such data objects, and all of them will be added to the response. * * @author bratseth - * @since 5.1.23 */ public class ResponseHeaders extends AbstractData { diff --git a/container-core/src/main/java/com/yahoo/processing/processors/RequestPropertyTracer.java b/container-core/src/main/java/com/yahoo/processing/processors/RequestPropertyTracer.java index e08cf013c19..a42b027b795 100644 --- a/container-core/src/main/java/com/yahoo/processing/processors/RequestPropertyTracer.java +++ b/container-core/src/main/java/com/yahoo/processing/processors/RequestPropertyTracer.java @@ -12,8 +12,7 @@ import java.util.Map; * A processor which adds the current content of the Request.properties() to * the trace before calling the next processor, if traceLevel is 4 or more. * - * @author bratseth - * @since 5.1.17 + * @author bratseth */ public class RequestPropertyTracer extends Processor { diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousRenderer.java index b6298558ff9..eb39c4c8117 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousRenderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousRenderer.java @@ -20,8 +20,7 @@ public abstract class AsynchronousRenderer <RESPONSE extends Response> extends R * Exposes JDisc wiring to ensure asynchronous cleanup. * * @param channel the channel to the client receiving the response - * @param completionHandler the JDisc completion handler which will be invoked at the end - * of the rendering + * @param completionHandler the JDisc completion handler which will be invoked at the end of the rendering * @throws IllegalStateException if attempted invoked more than once */ public abstract void setNetworkWiring(ContentChannel channel, CompletionHandler completionHandler); diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java index fdc3b63fc92..f86cad7c619 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java @@ -53,10 +53,8 @@ public abstract class AsynchronousSectionedRenderer<RESPONSE extends Response> e * stream to be used throughput the rendering. Subsequent calls must use the * same stream. * - * @param stream - * the stream to render to in this and all subsequent calls. - * @throws IOException - * passed on from the stream + * @param stream the stream to render to in this and all subsequent calls. + * @throws IOException passed on from the stream */ public abstract void beginResponse(OutputStream stream) throws IOException; diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/ProcessingRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/ProcessingRenderer.java index 556eec9b460..052c1c3f7f5 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/ProcessingRenderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/ProcessingRenderer.java @@ -167,7 +167,7 @@ public class ProcessingRenderer extends AsynchronousSectionedRenderer<Response> private static class TraceRenderingVisitor extends TraceVisitor { - private JSONWriter jsonWriter; + private final JSONWriter jsonWriter; public TraceRenderingVisitor(JSONWriter jsonWriter) { this.jsonWriter = jsonWriter; @@ -226,4 +226,5 @@ public class ProcessingRenderer extends AsynchronousSectionedRenderer<Response> return (IOException) super.getCause(); } } + } diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/Renderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/Renderer.java index d04eda943af..a7fa557f71a 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/Renderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/Renderer.java @@ -46,16 +46,11 @@ public abstract class Renderer<RESPONSE extends Response> extends AbstractCompon * exception causing failure wrapped in an ExecutionException if rendering * was not successful. * - * @param stream - * a stream API bridge to JDisc - * @param response - * the response to render - * @param execution - * the execution which created this response - * @param request - * the request matching the response - * @return a ListenableFuture containing a boolean where true indicates a - * successful rendering + * @param stream a stream API bridge to JDisc + * @param response the response to render + * @param execution the execution which created this response + * @param request the request matching the response + * @return a ListenableFuture containing a boolean where true indicates a successful rendering */ public abstract ListenableFuture<Boolean> render(OutputStream stream, RESPONSE response, Execution execution, Request request); @@ -63,17 +58,14 @@ public abstract class Renderer<RESPONSE extends Response> extends AbstractCompon /** * Name of the output encoding, if applicable. * - *<p>TODO: ensure null is OK - * - * @return The encoding of the output if applicable, e.g. "utf-8" + * @return the encoding of the output if applicable, e.g. "utf-8" */ public abstract String getEncoding(); /** * The MIME type of the rendered content sent to the client. * - * @return The mime type of the data written to the writer, e.g. - * "text/plain" + * @return The mime type of the data written to the writer, e.g. "text/plain" */ public abstract String getMimeType(); diff --git a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java index 94785819aa6..0a2c08530aa 100644 --- a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java @@ -19,8 +19,8 @@ import java.util.logging.Logger; public class JacksonJsonResponse<T> extends HttpResponse { private static final Logger log = Logger.getLogger(JacksonJsonResponse.class.getName()); - private static final ObjectMapper defaultJsonMapper = new ObjectMapper() - .registerModule(new JavaTimeModule()).registerModule(new Jdk8Module()); + private static final ObjectMapper defaultJsonMapper = + new ObjectMapper().registerModule(new JavaTimeModule()).registerModule(new Jdk8Module()); private final ObjectMapper jsonMapper; private final T entity; @@ -48,4 +48,5 @@ public class JacksonJsonResponse<T> extends HttpResponse { @Override public String getContentType() { return "application/json"; } public T getEntity() { return entity; } + } diff --git a/container-core/src/main/java/com/yahoo/restapi/Path.java b/container-core/src/main/java/com/yahoo/restapi/Path.java index fe65245fd15..23a791e7532 100644 --- a/container-core/src/main/java/com/yahoo/restapi/Path.java +++ b/container-core/src/main/java/com/yahoo/restapi/Path.java @@ -42,24 +42,6 @@ public class Path { private final Map<String, String> values = new HashMap<>(); private String rest = ""; - /** - * @deprecated use {@link #Path(URI)} for correct handling of URL encoded paths. - */ - @Deprecated - public Path(String path) { - this(path, ""); - } - - /** - * @deprecated use {@link #Path(URI, String)} for correct handling of URL encoded paths. - */ - @Deprecated - public Path(String path, String optionalPrefix) { - this.optionalPrefix = optionalPrefix; - this.pathString = path; - this.elements = path.split("/"); - } - public Path(URI uri) { this(uri, ""); } diff --git a/container-disc/src/main/java/com/yahoo/container/FilterConfigProvider.java b/container-disc/src/main/java/com/yahoo/container/FilterConfigProvider.java index c17b9d445a2..b22a8314d2b 100644 --- a/container-disc/src/main/java/com/yahoo/container/FilterConfigProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/FilterConfigProvider.java @@ -74,4 +74,5 @@ public final class FilterConfigProvider implements Provider<FilterConfig> { @Override public void deconstruct() {} + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java index 0e3110e26a8..09febdabc60 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java @@ -14,6 +14,7 @@ import java.util.List; * @author mortent */ public class AthenzIdentityProviderProvider implements Provider<AthenzIdentityProvider> { + private static final ThrowingAthenzIdentityProvider instance = new ThrowingAthenzIdentityProvider(); @Override @@ -84,4 +85,5 @@ public class AthenzIdentityProviderProvider implements Provider<AthenzIdentityPr throw new UnsupportedOperationException(message); } } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 4614f8f9857..3158c06b0b1 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -261,12 +261,16 @@ public final class ConfiguredApplication implements Application { private void startReconfigurerThread() { reconfigurerThread = new Thread(() -> { + boolean restartOnDeploy = false; while ( ! Thread.interrupted()) { try { ContainerBuilder builder = createBuilderWithGuiceBindings(); + // Restart on deploy is sticky: Once it is set no future generation should be applied until restart + restartOnDeploy = restartOnDeploy || qrConfig.restartOnDeploy(); + // Block until new config arrives, and it should be applied - configurer.getNewComponentGraph(builder.guiceModules().activate(), qrConfig.restartOnDeploy()); + configurer.getNewComponentGraph(builder.guiceModules().activate(), restartOnDeploy); initializeAndActivateContainer(builder); } catch (ConfigInterruptedException e) { break; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/RestrictedBundleContext.java b/container-disc/src/main/java/com/yahoo/container/jdisc/RestrictedBundleContext.java index a8acaf7dd10..baf7ac8c4dc 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/RestrictedBundleContext.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/RestrictedBundleContext.java @@ -22,25 +22,19 @@ public class RestrictedBundleContext implements BundleContext { @Override public ServiceRegistration<?> registerService(String[] strings, Object o, Dictionary<String, ?> stringDictionary) { - if (wrapped == null) { - return null; - } + if (wrapped == null) return null; return wrapped.registerService(strings, o, stringDictionary); } @Override public ServiceRegistration<?> registerService(String localHostname, Object o, Dictionary<String, ?> stringDictionary) { - if (wrapped == null) { - return null; - } + if (wrapped == null) return null; return wrapped.registerService(localHostname, o, stringDictionary); } @Override public <S> ServiceRegistration<S> registerService(Class<S> sClass, S s, Dictionary<String, ?> stringDictionary) { - if (wrapped == null) { - return null; - } + if (wrapped == null) return null; return wrapped.registerService(sClass, s, stringDictionary); } @@ -51,57 +45,43 @@ public class RestrictedBundleContext implements BundleContext { @Override public ServiceReference<?>[] getServiceReferences(String localHostname, String localHostname2) throws InvalidSyntaxException { - if (wrapped == null) { - return new ServiceReference<?>[0]; - } + if (wrapped == null) return new ServiceReference<?>[0]; return wrapped.getServiceReferences(localHostname, localHostname2); } @Override public ServiceReference<?>[] getAllServiceReferences(String localHostname, String localHostname2) throws InvalidSyntaxException { - if (wrapped == null) { - return new ServiceReference<?>[0]; - } + if (wrapped == null) return new ServiceReference<?>[0]; return wrapped.getAllServiceReferences(localHostname, localHostname2); } @Override public ServiceReference<?> getServiceReference(String localHostname) { - if (wrapped == null) { - return null; - } + if (wrapped == null) return null; return wrapped.getServiceReference(localHostname); } @Override public <S> ServiceReference<S> getServiceReference(Class<S> sClass) { - if (wrapped == null) { - return null; - } + if (wrapped == null) return null; return wrapped.getServiceReference(sClass); } @Override public <S> Collection<ServiceReference<S>> getServiceReferences(Class<S> sClass, String localHostname) throws InvalidSyntaxException { - if (wrapped == null) { - return Collections.<ServiceReference<S>>emptyList(); - } + if (wrapped == null) return Collections.<ServiceReference<S>>emptyList(); return wrapped.getServiceReferences(sClass, localHostname); } @Override public <S> S getService(ServiceReference<S> sServiceReference) { - if (wrapped == null) { - return null; - } + if (wrapped == null) return null; return wrapped.getService(sServiceReference); } @Override public boolean ungetService(ServiceReference<?> serviceReference) { - if (wrapped == null) { - return false; - } + if (wrapped == null) return false; return wrapped.ungetService(serviceReference); } @@ -110,10 +90,6 @@ public class RestrictedBundleContext implements BundleContext { return null; } - - //--------------------- - - @Override public String getProperty(String localHostname) { throw newException(); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/SecretStoreProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/SecretStoreProvider.java index 9c1dd00fdd4..eca62bc6ae6 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/SecretStoreProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/SecretStoreProvider.java @@ -34,4 +34,5 @@ public class SecretStoreProvider implements Provider<SecretStore> { throw new SecretNotFoundException("A secret store is not available"); } } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java index 0bb3832ddf5..b25517ec1f7 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/SystemInfoProvider.java @@ -19,9 +19,12 @@ public class SystemInfoProvider extends AbstractComponent implements Provider<Sy private final SystemInfo instance; - @Inject public SystemInfoProvider(ConfigserverConfig config) { + @Inject + public SystemInfoProvider(ConfigserverConfig config) { this.instance = new SystemInfo(new Zone(Environment.valueOf(config.environment()), config.region())); } - @Override public SystemInfo get() { return instance; } + @Override + public SystemInfo get() { return instance; } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java index 10bf96749e8..26014dba08c 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java @@ -11,6 +11,7 @@ import java.util.List; * @author mortent */ public interface AthenzIdentityProvider { + String domain(); String service(); SSLContext getIdentitySslContext(); @@ -22,4 +23,5 @@ public interface AthenzIdentityProvider { List<X509Certificate> getIdentityCertificate(); PrivateKey getPrivateKey(); Path trustStorePath(); + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProviderException.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProviderException.java index fd5839bfc45..039a0535c32 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProviderException.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProviderException.java @@ -13,4 +13,5 @@ public class AthenzIdentityProviderException extends RuntimeException { public AthenzIdentityProviderException(String message, Throwable cause) { super(message, cause); } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/DisableGuiceMetric.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/DisableGuiceMetric.java index 5c205cd5157..b168b21ac1c 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/DisableGuiceMetric.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/DisableGuiceMetric.java @@ -28,4 +28,5 @@ public class DisableGuiceMetric implements Metric { private static RuntimeException newException() { return new UnsupportedOperationException("The Metric framework is only available to components."); } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/ForwardingMetricConsumer.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/ForwardingMetricConsumer.java index c9caaa9d4b5..60b7b0335bf 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/ForwardingMetricConsumer.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/ForwardingMetricConsumer.java @@ -55,4 +55,5 @@ public final class ForwardingMetricConsumer implements MetricConsumer { this.contexts = contexts; } } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java index c452fb2435d..17effe2136a 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java @@ -17,6 +17,7 @@ import java.util.Map; * @author ollivir */ public class GarbageCollectionMetrics { + private static final String GC_COUNT = "jdisc.gc.count"; private static final String GC_TIME = "jdisc.gc.ms"; private static final String DIMENSION_KEY = "gcName"; @@ -35,7 +36,7 @@ public class GarbageCollectionMetrics { } } - private Map<String, LinkedList<GcStats>> gcStatistics; + private final Map<String, LinkedList<GcStats>> gcStatistics; private final Clock clock; @@ -92,4 +93,5 @@ public class GarbageCollectionMetrics { Map<String, LinkedList<GcStats>> getGcStatistics() { return gcStatistics; } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java index b29d7fe1f21..22a335a8171 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java @@ -38,4 +38,5 @@ class JrtMetrics { metric.add(metricName, countIncrement, null); } } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java index 05c3b88b788..0c26d6eefd9 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java @@ -20,7 +20,6 @@ import java.util.TimerTask; * * @author bjorncs * @author vegardh - * */ public class MetricUpdater extends AbstractComponent { @@ -139,5 +138,6 @@ public class MetricUpdater extends AbstractComponent { void schedule(Runnable runnable, Duration frequency); void cancel(); } + } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretStore.java b/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretStore.java index 7cd8e11c677..8af1f9860bf 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretStore.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretStore.java @@ -8,6 +8,7 @@ import java.util.List; * @author mortent */ public interface SecretStore { + /** Returns the secret for this key */ String getSecret(String key); @@ -18,4 +19,5 @@ public interface SecretStore { default List<Integer> listSecretVersions(String key) { throw new UnsupportedOperationException("Secret store does not support listing versions"); } + } diff --git a/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java b/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java index ae65fc3ad68..709441999d0 100644 --- a/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java +++ b/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java @@ -57,7 +57,7 @@ public class BindingsOverviewHandler extends AbstractRequestHandler { @Override protected com.yahoo.jdisc.Response newResponse() { com.yahoo.jdisc.Response response = new com.yahoo.jdisc.Response(statusToReturn); - response.headers().add("Content-Type", Arrays.asList(new String[]{"application/json"})); + response.headers().add("Content-Type", List.of("application/json")); return response; } }.connect(handler)); @@ -110,12 +110,10 @@ public class BindingsOverviewHandler extends AbstractRequestHandler { } private static JSONArray renderBindings(List<String> bindings) { - JSONArray ret = new JSONArray(); - + JSONArray array = new JSONArray(); for (String binding : bindings) - ret.put(binding); - - return ret; + array.put(binding); + return array; } private static JSONObject renderComponent(Object component, ComponentId id) { @@ -136,9 +134,9 @@ public class BindingsOverviewHandler extends AbstractRequestHandler { try { Bundle bundle = FrameworkUtil.getBundle(component.getClass()); - String bundleName = bundle != null ? - bundle.getSymbolicName() + ":" + bundle.getVersion() : - "From classpath"; + String bundleName = bundle != null + ? bundle.getSymbolicName() + ":" + bundle.getVersion() + : "From classpath"; return new BundleInfo(component.getClass().getName(), bundleName); } catch (Exception | NoClassDefFoundError e) { return new BundleInfo("Unavailable, reconfiguration in progress.", ""); @@ -155,12 +153,15 @@ public class BindingsOverviewHandler extends AbstractRequestHandler { } static final class BundleInfo { + public final String className; public final String bundleName; + BundleInfo(String className, String bundleName) { this.className = className; this.bundleName = bundleName; } + } static final class StatusResponse { @@ -182,7 +183,8 @@ public class BindingsOverviewHandler extends AbstractRequestHandler { } - private class IgnoredContent implements ContentChannel { + private static class IgnoredContent implements ContentChannel { + @Override public void write(ByteBuffer buf, CompletionHandler handler) { handler.completed(); @@ -192,5 +194,7 @@ public class BindingsOverviewHandler extends AbstractRequestHandler { public void close(CompletionHandler handler) { handler.completed(); } + } + } diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java index b90f96fb240..2b86d83547e 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusClientProvider.java @@ -61,4 +61,5 @@ public class MbusClientProvider implements Provider<MbusClient> { public void deconstruct() { client.release(); } + } diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index e62f6a8a21a..68b1f5aa5db 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -159,7 +159,7 @@ public final class SessionCache extends AbstractComponent { return sourcesCreator.retain(sourceLock, sources, p); } - private abstract class SessionCreator<PARAMS, KEY, SESSION extends SharedResource> { + private abstract static class SessionCreator<PARAMS, KEY, SESSION extends SharedResource> { abstract SESSION create(PARAMS p); @@ -352,6 +352,7 @@ public final class SessionCache extends AbstractComponent { } static class UnknownThrottlePolicySignature extends ThrottlePolicySignature { + private final ThrottlePolicy policy; UnknownThrottlePolicySignature(final ThrottlePolicy policy) { @@ -409,26 +410,16 @@ public final class SessionCache extends AbstractComponent { @Override public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; SourceSessionKey other = (SourceSessionKey) obj; if (policy == null) { - if (other.policy != null) { - return false; - } + if (other.policy != null) return false; } else if (!policy.equals(other.policy)) { return false; } - if (Double.doubleToLongBits(timeout) != Double.doubleToLongBits(other.timeout)) { - return false; - } + if (Double.doubleToLongBits(timeout) != Double.doubleToLongBits(other.timeout)) return false; return true; } } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 19fb1862262..cd3f011352d 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -6089,6 +6089,7 @@ "public final java.lang.Object get(java.lang.String, java.util.Map)", "public final java.lang.Object get(java.lang.String, java.util.Map, com.yahoo.processing.request.Properties)", "public final java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", + "public final com.yahoo.search.query.profile.compiled.DimensionalMap getEntries()", "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile clone()", "public java.lang.String toString()", "public bridge synthetic com.yahoo.component.AbstractComponent clone()", diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 4995927f7a2..ce31b9a3ba3 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -181,7 +181,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { //---------------- Tracing ---------------------------------------------------- - private static Logger log = Logger.getLogger(Query.class.getName()); + private static final Logger log = Logger.getLogger(Query.class.getName()); /** The time this query was created */ private long startTime; @@ -200,7 +200,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public static final CompoundName TIMEOUT = new CompoundName("timeout"); - private static QueryProfileType argumentType; + private static final QueryProfileType argumentType; static { argumentType = new QueryProfileType("native"); argumentType.setBuiltin(true); @@ -226,7 +226,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public static QueryProfileType getArgumentType() { return argumentType; } /** The aliases of query properties */ - private static Map<String, CompoundName> propertyAliases; + private static final Map<String, CompoundName> propertyAliases; static { Map<String,CompoundName> propertyAliasesBuilder = new HashMap<>(); addAliases(Query.getArgumentType(), propertyAliasesBuilder); @@ -316,7 +316,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * Creates a query from a request * * @param request the HTTP request from which this is created - * @param queryProfile the query profile to use for this query, or null if none. + * @param queryProfile the query profile to use for this query, or null if none */ public Query(HttpRequest request, CompiledQueryProfile queryProfile) { this(request, request.propertyMap(), queryProfile); @@ -325,9 +325,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** * Creates a query from a request * - * @param request the HTTP request from which this is created. - * @param requestMap the property map of the query. - * @param queryProfile the query profile to use for this query, or null if none. + * @param request the HTTP request from which this is created + * @param requestMap the property map of the query + * @param queryProfile the query profile to use for this query, or null if none */ public Query(HttpRequest request, Map<String, String> requestMap, CompiledQueryProfile queryProfile) { super(new QueryPropertyAliases(propertyAliases)); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java index c6b0f4a533b..2439908183c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -183,6 +183,11 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable return substitute(value.value(), context, substitution); } + /** Returns all the entries from the profile **/ + public final DimensionalMap<ValueWithSource> getEntries() { + return this.entries; + } + private Object substitute(Object value, Map<String, String> context, Properties substitution) { if (value == null) return value; if (substitution == null) return value; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java index faa2c39ee65..799bc814abe 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java @@ -14,7 +14,6 @@ import java.util.List; public class RefeedAction { public final String name; - public final boolean allowed; public final String documentType; public final String clusterName; public final List<ServiceInfo> services; @@ -22,13 +21,11 @@ public class RefeedAction { @JsonCreator public RefeedAction(@JsonProperty("name") String name, - @JsonProperty("allowed") boolean allowed, @JsonProperty("documentType") String documentType, @JsonProperty("clusterName") String clusterName, @JsonProperty("services") List<ServiceInfo> services, @JsonProperty("messages") List<String> messages) { this.name = name; - this.allowed = allowed; this.documentType = documentType; this.clusterName = clusterName; this.services = services; @@ -39,7 +36,6 @@ public class RefeedAction { public String toString() { return "RefeedAction{" + "name='" + name + '\'' + - ", allowed=" + allowed + ", documentType='" + documentType + '\'' + ", clusterName='" + clusterName + '\'' + ", services=" + services + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ReindexAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ReindexAction.java index c5735fbd4a6..c2b28a94c66 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ReindexAction.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ReindexAction.java @@ -14,7 +14,6 @@ import java.util.List; public class ReindexAction { public final String name; - public final boolean allowed; public final String documentType; public final String clusterName; public final List<ServiceInfo> services; @@ -22,13 +21,11 @@ public class ReindexAction { @JsonCreator public ReindexAction(@JsonProperty("name") String name, - @JsonProperty("allowed") boolean allowed, @JsonProperty("documentType") String documentType, @JsonProperty("clusterName") String clusterName, @JsonProperty("services") List<ServiceInfo> services, @JsonProperty("messages") List<String> messages) { this.name = name; - this.allowed = allowed; this.documentType = documentType; this.clusterName = clusterName; this.services = services; @@ -39,7 +36,6 @@ public class ReindexAction { public String toString() { return "ReindexAction{" + "name='" + name + '\'' + - ", allowed=" + allowed + ", documentType='" + documentType + '\'' + ", clusterName='" + clusterName + '\'' + ", services=" + services + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java index fd339e3bb43..98b7ffd1d47 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Cluster.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import java.time.Instant; +import java.util.List; import java.util.Optional; /** @@ -17,19 +19,25 @@ public class Cluster { private final ClusterResources current; private final Optional<ClusterResources> target; private final Optional<ClusterResources> suggested; + private final List<ScalingEvent> scalingEvents; + private final String autoscalingStatus; public Cluster(ClusterSpec.Id id, ClusterResources min, ClusterResources max, ClusterResources current, Optional<ClusterResources> target, - Optional<ClusterResources> suggested) { + Optional<ClusterResources> suggested, + List<ScalingEvent> scalingEvents, + String autoscalingStatus) { this.id = id; this.min = min; this.max = max; this.current = current; this.target = target; this.suggested = suggested; + this.scalingEvents = scalingEvents; + this.autoscalingStatus = autoscalingStatus; } public ClusterSpec.Id id() { return id; } @@ -38,10 +46,29 @@ public class Cluster { public ClusterResources current() { return current; } public Optional<ClusterResources> target() { return target; } public Optional<ClusterResources> suggested() { return suggested; } + public List<ScalingEvent> scalingEvents() { return scalingEvents; } + public String autoscalingStatus() { return autoscalingStatus; } @Override public String toString() { return "cluster '" + id + "'"; } + public static class ScalingEvent { + + private final ClusterResources from, to; + private final Instant at; + + public ScalingEvent(ClusterResources from, ClusterResources to, Instant at) { + this.from = from; + this.to = to; + this.at = at; + } + + public ClusterResources from() { return from; } + public ClusterResources to() { return to; } + public Instant at() { return at; } + + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index 8fd294f64f8..7d85c11789b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -14,10 +14,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeHist import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * A node in hosted Vespa. @@ -57,6 +59,8 @@ public class Node { private final Optional<ApplicationId> exclusiveTo; private final Map<String, JsonNode> reports; private final List<NodeHistory> history; + private final Set<String> additionalIpAddresses; + private final String openStackId; public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, @@ -64,7 +68,8 @@ public class Node { Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, int cost, String flavor, String clusterId, ClusterType clusterType, boolean wantToRetire, boolean wantToDeprovision, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, - DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, JsonNode> reports, List<NodeHistory> history) { + DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, JsonNode> reports, List<NodeHistory> history, + Set<String> additionalIpAddresses, String openStackId) { this.hostname = hostname; this.parentHostname = parentHostname; this.state = state; @@ -95,6 +100,8 @@ public class Node { this.currentDockerImage = currentDockerImage; this.reports = reports; this.history = history; + this.openStackId = openStackId; + this.additionalIpAddresses = additionalIpAddresses; } public HostName hostname() { @@ -211,6 +218,14 @@ public class Node { return history; } + public Set<String> additionalIpAddresses() { + return additionalIpAddresses; + } + + public String openStackId() { + return openStackId; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -285,6 +300,8 @@ public class Node { private Optional<ApplicationId> exclusiveTo = Optional.empty(); private Map<String, JsonNode> reports = new HashMap<>(); private List<NodeHistory> history = new ArrayList<>(); + private Set<String> additionalIpAddresses = new HashSet<>(); + private String openStackId; public Builder() { } @@ -319,6 +336,8 @@ public class Node { this.exclusiveTo = node.exclusiveTo; this.reports = node.reports; this.history = node.history; + this.additionalIpAddresses = node.additionalIpAddresses; + this.openStackId = node.openStackId; } public Builder hostname(HostName hostname) { @@ -466,12 +485,22 @@ public class Node { return this; } + public Builder additionalIpAddresses(Set<String> additionalIpAddresses) { + this.additionalIpAddresses = additionalIpAddresses; + return this; + } + + public Builder openStackId(String openStackId) { + this.openStackId = openStackId; + return this; + } + public Node build() { return new Node(hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState, suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, cost, flavor, clusterId, clusterType, wantToRetire, wantToDeprovision, reservedTo, exclusiveTo, - wantedDockerImage, currentDockerImage, reports, history); + wantedDockerImage, currentDockerImage, reports, history, additionalIpAddresses, openStackId); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index ca8af48e4fd..af1b3fa53fc 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -135,7 +135,9 @@ public interface NodeRepository { dockerImageFrom(node.getWantedDockerImage()), dockerImageFrom(node.getCurrentDockerImage()), node.getReports(), - node.getHistory()); + node.getHistory(), + node.getAdditionalIpAddresses(), + node.getOpenStackId()); } private static String clusterIdOf(NodeMembership nodeMembership) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java index 6054e05149b..5c946538625 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java @@ -15,7 +15,6 @@ import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) public class PrepareResponse { public TenantId tenant; - @JsonProperty("activate") public URI activationUri; public String message; public List<Log> log; public ConfigChangeActions configChangeActions; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 700be6d263a..efdeff8fc16 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -144,6 +144,9 @@ public enum JobType { Map.of(Public, ZoneId.from("dev", "aws-us-east-1c"), PublicCd, ZoneId.from("dev", "aws-us-east-1c"))), + perfAwsUsEast1c ("perf-aws-us-east-1c", + Map.of(Public, ZoneId.from("perf", "aws-us-east-1c"))), + perfUsEast3 ("perf-us-east-3", Map.of(main, ZoneId.from("perf" , "us-east-3"))); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java index 298928a881d..99a72fbc827 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ClusterData.java @@ -7,7 +7,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * @author bratseth @@ -26,6 +28,10 @@ public class ClusterData { public ClusterResourcesData suggested; @JsonProperty("target") public ClusterResourcesData target; + @JsonProperty("scalingEvents") + public List<ScalingEventData> scalingEvents; + @JsonProperty("autoscalingStatus") + public String autoscalingStatus; public Cluster toCluster(String id) { return new Cluster(ClusterSpec.Id.from(id), @@ -33,7 +39,10 @@ public class ClusterData { max.toClusterResources(), current.toClusterResources(), target == null ? Optional.empty() : Optional.of(target.toClusterResources()), - suggested == null ? Optional.empty() : Optional.of(suggested.toClusterResources())); + suggested == null ? Optional.empty() : Optional.of(suggested.toClusterResources()), + scalingEvents == null ? List.of() + : scalingEvents.stream().map(data -> data.toScalingEvent()).collect(Collectors.toList()), + autoscalingStatus); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java index 97e7f2e897a..393814478dd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java @@ -13,6 +13,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) public class NodeHistory { + @JsonProperty("at") public Long at; @JsonProperty("agent") diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java index 7bb47185751..65d6f2a5fa6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; @@ -29,6 +28,8 @@ public class NodeRepositoryNode { private Set<String> ipAddresses; @JsonProperty("additionalIpAddresses") private Set<String> additionalIpAddresses; + @JsonProperty("additionalHostnames") + private List<String> additionalHostnames; @JsonProperty("openStackId") private String openStackId; @JsonProperty("flavor") @@ -142,6 +143,14 @@ public class NodeRepositoryNode { this.additionalIpAddresses = additionalIpAddresses; } + public List<String> getAdditionalHostnames() { + return additionalHostnames; + } + + public void setAdditionalHostnames(List<String> additionalHostnames) { + this.additionalHostnames = additionalHostnames; + } + public String getOpenStackId() { return openStackId; } @@ -397,6 +406,7 @@ public class NodeRepositoryNode { ", hostname='" + hostname + '\'' + ", ipAddresses=" + ipAddresses + ", additionalIpAddresses=" + additionalIpAddresses + + ", additionalHostnames=" + additionalHostnames + ", openStackId='" + openStackId + '\'' + ", flavor='" + flavor + '\'' + ", resources=" + resources + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ScalingEventData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ScalingEventData.java new file mode 100644 index 00000000000..b33a7436522 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ScalingEventData.java @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.noderepository; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; + +import java.time.Instant; + +/** + * @author bratseth + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ScalingEventData { + + @JsonProperty("from") + public ClusterResourcesData from; + + @JsonProperty("to") + public ClusterResourcesData to; + + @JsonProperty("at") + public Long at; + + public Cluster.ScalingEvent toScalingEvent() { + return new Cluster.ScalingEvent(from.toClusterResources(), to.toClusterResources(), Instant.ofEpochMilli(at)); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index c3126cc8b7a..85db447dfbd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -92,7 +92,8 @@ public enum RoleDefinition { paymentProcessor(Policy.paymentProcessor), hostedAccountant(Policy.hostedAccountant, - Policy.collectionMethodUpdate); + Policy.collectionMethodUpdate, + Policy.planUpdate); private final Set<RoleDefinition> parents; private final Set<Policy> policies; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 9b959bf1765..ed1e442f266 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -219,43 +219,11 @@ public class InternalStepRunner implements StepRunner { LogEntry.typeOf(LogLevel.parse(entry.level)), entry.message)) .collect(toList())); - if ( ! prepareResponse.configChangeActions.refeedActions.stream().allMatch(action -> action.allowed)) { - List<String> messages = new ArrayList<>(); - messages.add("Deploy failed due to non-compatible changes that require re-feed."); - messages.add("Your options are:"); - messages.add("1. Revert the incompatible changes."); - messages.add("2. If you think it is safe in your case, you can override this validation, see"); - messages.add(" http://docs.vespa.ai/documentation/reference/validation-overrides.html"); - messages.add("3. Deploy as a new application under a different name."); - messages.add("Illegal actions:"); - prepareResponse.configChangeActions.refeedActions.stream() - .filter(action -> ! action.allowed) - .flatMap(action -> action.messages.stream()) - .forEach(messages::add); - logger.log(messages); - return Optional.of(deploymentFailed); - } - - if ( ! prepareResponse.configChangeActions.reindexActions.stream().allMatch(action -> action.allowed)) { - List<String> messages = new ArrayList<>(); - messages.add("Deploy failed due to non-compatible changes that require re-index."); - messages.add("Your options are:"); - messages.add("1. Revert the incompatible changes."); - messages.add("2. If you think it is safe in your case, you can override this validation, see"); - messages.add(" http://docs.vespa.ai/documentation/reference/validation-overrides.html"); - messages.add("3. Deploy as a new application under a different name."); - messages.add("Illegal actions:"); - prepareResponse.configChangeActions.reindexActions.stream() - .filter(action -> ! action.allowed) - .flatMap(action -> action.messages.stream()) - .forEach(messages::add); - logger.log(messages); - return Optional.of(deploymentFailed); - } logger.log("Deployment successful."); if (prepareResponse.message != null) logger.log(prepareResponse.message); + return Optional.of(running); } catch (ConfigServerException e) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 1c0dca26f5a..740a70fc6d1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -701,6 +701,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { toSlime(cluster.current(), clusterObject.setObject("current")); cluster.target().ifPresent(target -> toSlime(target, clusterObject.setObject("target"))); cluster.suggested().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject("suggested"))); + scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); + clusterObject.setString("autoscalingStatus", cluster.autoscalingStatus()); } return new SlimeJsonResponse(slime); } @@ -1539,11 +1541,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse reindex(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); ZoneId zone = ZoneId.from(environment, region); - List<String> clusterNames = Optional.ofNullable(request.getProperty("cluster")).stream() + List<String> clusterNames = Optional.ofNullable(request.getProperty("clusterId")).stream() .flatMap(clusters -> Stream.of(clusters.split(","))) .filter(cluster -> ! cluster.isBlank()) .collect(toUnmodifiableList()); - List<String> documentTypes = Optional.ofNullable(request.getProperty("type")).stream() + List<String> documentTypes = Optional.ofNullable(request.getProperty("documentType")).stream() .flatMap(types -> Stream.of(types.split(","))) .filter(type -> ! type.isBlank()) .collect(toUnmodifiableList()); @@ -1914,6 +1916,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { object.setDouble("cost", Math.round(resources.nodes() * resources.nodeResources().cost() * 100.0 / 3.0) / 100.0); } + private void scalingEventsToSlime(List<Cluster.ScalingEvent> scalingEvents, Cursor scalingEventsArray) { + for (Cluster.ScalingEvent scalingEvent : scalingEvents) { + Cursor scalingEventObject = scalingEventsArray.addObject(); + toSlime(scalingEvent.from(), scalingEventObject.setObject("from")); + toSlime(scalingEvent.to(), scalingEventObject.setObject("to")); + scalingEventObject.setLong("at", scalingEvent.at().toEpochMilli()); + } + } + private void toSlime(NodeResources resources, Cursor object) { object.setDouble("vcpu", resources.vcpu()); object.setDouble("memoryGb", resources.memoryGb()); @@ -2054,7 +2065,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { for (RefeedAction refeedAction : result.prepareResponse().configChangeActions.refeedActions) { Cursor refeedActionObject = refeedActionsArray.addObject(); refeedActionObject.setString("name", refeedAction.name); - refeedActionObject.setBool("allowed", refeedAction.allowed); refeedActionObject.setString("documentType", refeedAction.documentType); refeedActionObject.setString("clusterName", refeedAction.clusterName); serviceInfosToSlime(refeedAction.services, refeedActionObject.setArray("services")); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index 199eee6d0c9..d6c6f5ff167 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -18,12 +18,14 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.TenantController; import com.yahoo.vespa.hosted.controller.api.integration.billing.CollectionMethod; import com.yahoo.vespa.hosted.controller.api.integration.billing.PaymentInstrument; import com.yahoo.vespa.hosted.controller.api.integration.billing.Invoice; import com.yahoo.vespa.hosted.controller.api.integration.billing.InstrumentOwner; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; import javax.ws.rs.BadRequestException; @@ -37,6 +39,7 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.concurrent.Executor; @@ -53,6 +56,7 @@ public class BillingApiHandler extends LoggingRequestHandler { private final BillingController billingController; private final ApplicationController applicationController; + private final TenantController tenantController; public BillingApiHandler(Executor executor, AccessLog accessLog, @@ -60,6 +64,7 @@ public class BillingApiHandler extends LoggingRequestHandler { super(executor, accessLog); this.billingController = controller.serviceRegistry().billingController(); this.applicationController = controller.applications(); + this.tenantController = controller.tenants(); } @Override @@ -175,15 +180,16 @@ public class BillingApiHandler extends LoggingRequestHandler { root.setString("until", untilDate.format(DateTimeFormatter.ISO_DATE)); var tenants = root.setArray("tenants"); - uncommittedInvoices.forEach((tenant, invoice) -> { + tenantController.asList().stream().sorted(Comparator.comparing(Tenant::name)).forEach(tenant -> { + var invoice = uncommittedInvoices.get(tenant.name()); var tc = tenants.addObject(); - tc.setString("tenant", tenant.value()); - getPlanForTenant(tc, tenant); - getCollectionForTenant(tc, tenant); + tc.setString("tenant", tenant.name().value()); + getPlanForTenant(tc, tenant.name()); + getCollectionForTenant(tc, tenant.name()); renderCurrentUsage(tc.setObject("current"), invoice); - renderAdditionalItems(tc.setObject("additional").setArray("items"), billingController.getUnusedLineItems(tenant)); + renderAdditionalItems(tc.setObject("additional").setArray("items"), billingController.getUnusedLineItems(tenant.name())); - billingController.getDefaultInstrument(tenant).ifPresent(card -> + billingController.getDefaultInstrument(tenant.name()).ifPresent(card -> renderInstrument(tc.setObject("payment"), card) ); }); @@ -302,6 +308,7 @@ public class BillingApiHandler extends LoggingRequestHandler { } private void renderCurrentUsage(Cursor cursor, Invoice currentUsage) { + if (currentUsage == null) return; cursor.setString("amount", currentUsage.sum().toPlainString()); cursor.setString("status", "accrued"); cursor.setString("from", currentUsage.getStartTime().format(DATE_TIME_FORMATTER)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 7b248052eac..d54971f5b1d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -125,34 +125,7 @@ public class InternalStepRunnerTest { } @Test - public void reindexRequirementBlocksDeployment() { - tester.configServer().setConfigChangeActions(new ConfigChangeActions(List.of(), - List.of(), - List.of(new ReindexAction("Reindex", - false, - "doctype", - "cluster", - Collections.emptyList(), - List.of("Reindex it!"))))); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); - } - - @Test - public void refeedRequirementBlocksDeployment() { - tester.configServer().setConfigChangeActions(new ConfigChangeActions(List.of(), - List.of(new RefeedAction("Refeed", - false, - "doctype", - "cluster", - Collections.emptyList(), - singletonList("Refeed it!"))), - List.of())); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); - } - - @Test + // TODO jonmv: Change to only wait for restarts, and remove triggering of restarts from runner. public void restartsServicesAndWaitsForRestartAndReboot() { RunId id = app.newRun(JobType.productionUsCentral1); ZoneId zone = id.type().zone(system()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 8acce352d5a..27b739160fc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -108,12 +108,17 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer /** Assigns a reserved tenant node to the given deployment, with initial versions. */ public void provision(ZoneId zone, ApplicationId application, ClusterSpec.Id clusterId) { + var current = new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1, slow, remote)); Cluster cluster = new Cluster(clusterId, new ClusterResources(2, 1, new NodeResources(1, 4, 20, 1, slow, remote)), new ClusterResources(2, 1, new NodeResources(4, 16, 90, 1, slow, remote)), - new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1, slow, remote)), + current, Optional.of(new ClusterResources(2, 1, new NodeResources(3, 8, 50, 1, slow, remote))), - Optional.empty()); + Optional.empty(), + List.of(new Cluster.ScalingEvent(new ClusterResources(0, 0, NodeResources.unspecified()), + current, + Instant.ofEpochMilli(1234))), + "the autoscaling status"); nodeRepository.putApplication(zone, new com.yahoo.vespa.hosted.controller.api.integration.configserver.Application(application, List.of(cluster))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 7e097304790..5e98ac0d3ee 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -584,14 +584,14 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST a 'reindex application' command with cluster filter tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/reindex", POST) - .properties(Map.of("cluster", "boo,moo")) + .properties(Map.of("clusterId", "boo,moo")) .userIdentity(USER_ID), "{\"message\":\"Requested reindexing of tenant1.application1.instance1 in prod.us-central-1, on clusters boo, moo\"}"); // POST a 'reindex application' command with cluster and document type filters tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/reindex", POST) - .properties(Map.of("cluster", "boo,moo", - "type", "foo,boo")) + .properties(Map.of("clusterId", "boo,moo", + "documentType", "foo,boo")) .userIdentity(USER_ID), "{\"message\":\"Requested reindexing of tenant1.application1.instance1 in prod.us-central-1, on clusters boo, moo, for types foo, boo\"}"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json index 65fa2a4bf70..817cee7732a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json @@ -52,7 +52,39 @@ "storageType": "remote" }, "cost": "(ignore)" - } + }, + "scalingEvents": [ + { + "from": { + "nodes": 0, + "groups": 0, + "nodeResources": { + "vcpu": 0.0, + "memoryGb": 0.0, + "diskGb": 0.0, + "bandwidthGbps": 0.0, + "diskSpeed": "fast", + "storageType": "any" + }, + "cost": "(ignore)" + }, + "to": { + "nodes": 2, + "groups": 1, + "nodeResources": { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "slow", + "storageType": "remote" + }, + "cost": "(ignore)" + }, + "at": 1234 + } + ], + "autoscalingStatus": "the autoscaling status" } } }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java index 8493250d9a3..1d0f0935c05 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.billing.CollectionMethod; import com.yahoo.vespa.hosted.controller.api.integration.billing.Invoice; import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; @@ -9,12 +10,17 @@ import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; +import com.yahoo.vespa.hosted.controller.security.Auth0Credentials; +import com.yahoo.vespa.hosted.controller.security.CloudTenantSpec; +import com.yahoo.vespa.hosted.controller.security.Credentials; +import com.yahoo.vespa.hosted.controller.security.TenantSpec; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.io.File; import java.math.BigDecimal; +import java.security.Principal; import java.time.LocalDate; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -165,6 +171,9 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest { @Test public void list_all_uninvoiced_items() { + tester.controller().tenants().create(new CloudTenantSpec(tenant, ""), new Auth0Credentials(() -> "foo", Set.of(Role.hostedOperator()))); + tester.controller().tenants().create(new CloudTenantSpec(tenant2, ""), new Auth0Credentials(() -> "foo", Set.of(Role.hostedOperator()))); + var invoice = createInvoice(); billingController.setPlan(tenant, PlanId.from("some-plan"), true); billingController.setPlan(tenant2, PlanId.from("some-plan"), true); @@ -172,7 +181,6 @@ public class BillingApiHandlerTest extends ControllerContainerCloudTest { billingController.addLineItem(tenant, "support", new BigDecimal("42"), "Smith"); billingController.addInvoice(tenant2, invoice, false); - var request = request("/billing/v1/billing?until=2020-05-28").roles(financeAdmin); tester.assertResponse(request, new File("billing-all-tenants")); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants index 81868a44e57..5c61dc6e86e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants @@ -2,7 +2,7 @@ "until":"2020-05-28", "tenants":[ { - "tenant":"tenant2", + "tenant":"tenant1", "plan":"some-plan", "planName":"Plan with id: some-plan", "collection": "AUTO", @@ -20,10 +20,21 @@ } ] }, - "additional":{"items":[]} + "additional": + { + "items":[ + { + "id":"line-item-id", + "description":"support", + "amount":"42.00", + "plan":"some-plan", + "planName":"Plan with id: some-plan" + } + ] + } }, { - "tenant":"tenant1", + "tenant":"tenant2", "plan":"some-plan", "planName":"Plan with id: some-plan", "collection": "AUTO", @@ -41,18 +52,7 @@ } ] }, - "additional": - { - "items":[ - { - "id":"line-item-id", - "description":"support", - "amount":"42.00", - "plan":"some-plan", - "planName":"Plan with id: some-plan" - } - ] - } + "additional":{"items":[]} } ] }
\ No newline at end of file diff --git a/document/abi-spec.json b/document/abi-spec.json index b119f9991b3..903b7a897df 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -1911,6 +1911,8 @@ "public java.lang.Class getValueClass()", "public boolean isValueCompatible(com.yahoo.document.datatypes.FieldValue)", "public com.yahoo.tensor.TensorType getTensorType()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", "public bridge synthetic com.yahoo.document.DataType clone()", "public bridge synthetic com.yahoo.vespa.objects.Identifiable clone()", "public bridge synthetic java.lang.Object clone()" diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index fa5dffd042a..104d63cae96 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -84,6 +84,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com this.dataTypeId = dataTypeId; } + @Override public DataType clone() { return (DataType)super.clone(); } @@ -248,14 +249,17 @@ public abstract class DataType extends Identifiable implements Serializable, Com manager.registerSingleType(this); } + @Override public int hashCode() { return name.hashCode(); } + @Override public boolean equals(Object other) { return (other instanceof DataType) && (dataTypeId == ((DataType)other).dataTypeId); } + @Override public String toString() { return "datatype " + name + " (code: " + dataTypeId + ")"; } diff --git a/document/src/main/java/com/yahoo/document/TensorDataType.java b/document/src/main/java/com/yahoo/document/TensorDataType.java index b21461597bf..c4fdff30f8b 100644 --- a/document/src/main/java/com/yahoo/document/TensorDataType.java +++ b/document/src/main/java/com/yahoo/document/TensorDataType.java @@ -6,6 +6,8 @@ import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.objects.Ids; +import java.util.Objects; + /** * A DataType containing a tensor type * @@ -23,6 +25,7 @@ public class TensorDataType extends DataType { this.tensorType = tensorType; } + @Override public TensorDataType clone() { return (TensorDataType)super.clone(); } @@ -48,4 +51,17 @@ public class TensorDataType extends DataType { /** Returns the type of the tensor this field can hold */ public TensorType getTensorType() { return tensorType; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + TensorDataType that = (TensorDataType) o; + return Objects.equals(tensorType, that.tensorType); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), tensorType); + } } diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.h b/document/src/vespa/document/serialization/vespadocumentdeserializer.h index 6792914d9da..5819c6a23cf 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.h +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.h @@ -7,7 +7,7 @@ #include <memory> namespace vespalib { class nbostream; } -namespace vespalib::eval { class Value; } +namespace vespalib::eval { struct Value; } namespace document { class DocumentId; diff --git a/eval/src/tests/eval/aggr/aggr_test.cpp b/eval/src/tests/eval/aggr/aggr_test.cpp index b3e9c625fd9..9045df68305 100644 --- a/eval/src/tests/eval/aggr/aggr_test.cpp +++ b/eval/src/tests/eval/aggr/aggr_test.cpp @@ -19,6 +19,36 @@ TEST("require that aggregator list returns appropriate entries") { EXPECT_EQUAL(int(list[6]), int(Aggr::MIN)); } +TEST("require that aggr::is_simple works as expected") { + EXPECT_FALSE(aggr::is_simple(Aggr::AVG)); + EXPECT_FALSE(aggr::is_simple(Aggr::COUNT)); + EXPECT_TRUE (aggr::is_simple(Aggr::PROD)); + EXPECT_TRUE (aggr::is_simple(Aggr::SUM)); + EXPECT_TRUE (aggr::is_simple(Aggr::MAX)); + EXPECT_FALSE(aggr::is_simple(Aggr::MEDIAN)); + EXPECT_TRUE (aggr::is_simple(Aggr::MIN)); +} + +TEST("require that aggr::is_ident works as expected") { + EXPECT_TRUE (aggr::is_ident(Aggr::AVG)); + EXPECT_FALSE(aggr::is_ident(Aggr::COUNT)); + EXPECT_TRUE (aggr::is_ident(Aggr::PROD)); + EXPECT_TRUE (aggr::is_ident(Aggr::SUM)); + EXPECT_TRUE (aggr::is_ident(Aggr::MAX)); + EXPECT_TRUE (aggr::is_ident(Aggr::MEDIAN)); + EXPECT_TRUE (aggr::is_ident(Aggr::MIN)); +} + +TEST("require that aggr::is_complex works as expected") { + EXPECT_FALSE(aggr::is_complex(Aggr::AVG)); + EXPECT_FALSE(aggr::is_complex(Aggr::COUNT)); + EXPECT_FALSE(aggr::is_complex(Aggr::PROD)); + EXPECT_FALSE(aggr::is_complex(Aggr::SUM)); + EXPECT_FALSE(aggr::is_complex(Aggr::MAX)); + EXPECT_TRUE (aggr::is_complex(Aggr::MEDIAN)); + EXPECT_FALSE(aggr::is_complex(Aggr::MIN)); +} + TEST("require that AVG aggregator works as expected") { Stash stash; Aggregator &aggr = Aggregator::create(Aggr::AVG, stash); @@ -28,6 +58,7 @@ TEST("require that AVG aggregator works as expected") { aggr.next(30.0), EXPECT_EQUAL(aggr.result(), 20.0); aggr.first(100.0), EXPECT_EQUAL(aggr.result(), 100.0); aggr.next(200.0), EXPECT_EQUAL(aggr.result(), 150.0); + EXPECT_TRUE(aggr.enum_value() == Aggr::AVG); } TEST("require that COUNT aggregator works as expected") { @@ -39,6 +70,7 @@ TEST("require that COUNT aggregator works as expected") { aggr.next(30.0), EXPECT_EQUAL(aggr.result(), 3.0); aggr.first(100.0), EXPECT_EQUAL(aggr.result(), 1.0); aggr.next(200.0), EXPECT_EQUAL(aggr.result(), 2.0); + EXPECT_TRUE(aggr.enum_value() == Aggr::COUNT); } TEST("require that PROD aggregator works as expected") { @@ -50,6 +82,13 @@ TEST("require that PROD aggregator works as expected") { aggr.next(30.0), EXPECT_EQUAL(aggr.result(), 6000.0); aggr.first(100.0), EXPECT_EQUAL(aggr.result(), 100.0); aggr.next(200.0), EXPECT_EQUAL(aggr.result(), 20000.0); + EXPECT_TRUE(aggr.enum_value() == Aggr::PROD); +} + +TEST("require that Prod combine works as expected") { + using Type = Prod<double>; + EXPECT_EQUAL(Type::combine(3,7), 21.0); + EXPECT_EQUAL(Type::combine(5,4), 20.0); } TEST("require that SUM aggregator works as expected") { @@ -61,6 +100,13 @@ TEST("require that SUM aggregator works as expected") { aggr.next(30.0), EXPECT_EQUAL(aggr.result(), 60.0); aggr.first(100.0), EXPECT_EQUAL(aggr.result(), 100.0); aggr.next(200.0), EXPECT_EQUAL(aggr.result(), 300.0); + EXPECT_TRUE(aggr.enum_value() == Aggr::SUM); +} + +TEST("require that Sum combine works as expected") { + using Type = Sum<double>; + EXPECT_EQUAL(Type::combine(3,7), 10.0); + EXPECT_EQUAL(Type::combine(5,4), 9.0); } TEST("require that MAX aggregator works as expected") { @@ -72,6 +118,13 @@ TEST("require that MAX aggregator works as expected") { aggr.next(30.0), EXPECT_EQUAL(aggr.result(), 30.0); aggr.first(100.0), EXPECT_EQUAL(aggr.result(), 100.0); aggr.next(200.0), EXPECT_EQUAL(aggr.result(), 200.0); + EXPECT_TRUE(aggr.enum_value() == Aggr::MAX); +} + +TEST("require that Max combine works as expected") { + using Type = Max<double>; + EXPECT_EQUAL(Type::combine(3,7), 7.0); + EXPECT_EQUAL(Type::combine(5,4), 5.0); } TEST("require that MEDIAN aggregator works as expected") { @@ -85,6 +138,7 @@ TEST("require that MEDIAN aggregator works as expected") { aggr.next(16.0), EXPECT_EQUAL(aggr.result(), 16.0); aggr.first(100.0), EXPECT_EQUAL(aggr.result(), 100.0); aggr.next(200.0), EXPECT_EQUAL(aggr.result(), 150.0); + EXPECT_TRUE(aggr.enum_value() == Aggr::MEDIAN); } TEST("require that MEDIAN aggregator handles NaN values") { @@ -108,6 +162,13 @@ TEST("require that MIN aggregator works as expected") { aggr.next(30.0), EXPECT_EQUAL(aggr.result(), 10.0); aggr.first(100.0), EXPECT_EQUAL(aggr.result(), 100.0); aggr.next(200.0), EXPECT_EQUAL(aggr.result(), 100.0); + EXPECT_TRUE(aggr.enum_value() == Aggr::MIN); +} + +TEST("require that Min combine works as expected") { + using Type = Min<double>; + EXPECT_EQUAL(Type::combine(3,7), 3.0); + EXPECT_EQUAL(Type::combine(5,4), 4.0); } template <template <typename T> typename A> diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp index 77801f44bb8..d58adbbcef0 100644 --- a/eval/src/tests/eval/value_type/value_type_test.cpp +++ b/eval/src/tests/eval/value_type/value_type_test.cpp @@ -8,8 +8,6 @@ using namespace vespalib::eval; -using CellType = ValueType::CellType; - const size_t npos = ValueType::Dimension::npos; ValueType type(const vespalib::string &type_str) { diff --git a/eval/src/tests/tensor/dense_single_reduce_function/dense_single_reduce_function_test.cpp b/eval/src/tests/tensor/dense_single_reduce_function/dense_single_reduce_function_test.cpp index 949c5277e18..d3495befe7e 100644 --- a/eval/src/tests/tensor/dense_single_reduce_function/dense_single_reduce_function_test.cpp +++ b/eval/src/tests/tensor/dense_single_reduce_function/dense_single_reduce_function_test.cpp @@ -26,6 +26,7 @@ const TensorEngine &prod_engine = DefaultTensorEngine::ref(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() .add_dense({{"a", 2}, {"b", 3}, {"c", 4}, {"d", 5}}) + .add_dense({{"a", 9}, {"b", 9}, {"c", 9}, {"d", 9}}) .add_cube("a", 2, "b", 1, "c", 1) .add_cube("a", 1, "b", 2, "c", 1) .add_cube("a", 1, "b", 1, "c", 2) @@ -36,17 +37,35 @@ EvalFixture::ParamRepo make_params() { } EvalFixture::ParamRepo param_repo = make_params(); -void verify_optimized(const vespalib::string &expr, size_t dim_idx, Aggr aggr) -{ +struct ReduceSpec { + size_t outer_size; + size_t reduce_size; + size_t inner_size; + Aggr aggr; +}; + +void verify_optimized_impl(const vespalib::string &expr, const std::vector<ReduceSpec> &spec_list) { EvalFixture slow_fixture(prod_engine, expr, param_repo, false); EvalFixture fixture(prod_engine, expr, param_repo, true); EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo)); EXPECT_EQUAL(fixture.result(), slow_fixture.result()); auto info = fixture.find_all<DenseSingleReduceFunction>(); - ASSERT_EQUAL(info.size(), 1u); - EXPECT_TRUE(info[0]->result_is_mutable()); - EXPECT_EQUAL(info[0]->dim_idx(), dim_idx); - EXPECT_EQUAL(int(info[0]->aggr()), int(aggr)); + ASSERT_EQUAL(info.size(), spec_list.size()); + for (size_t i = 0; i < spec_list.size(); ++i) { + EXPECT_TRUE(info[i]->result_is_mutable()); + EXPECT_EQUAL(info[i]->outer_size(), spec_list[i].outer_size); + EXPECT_EQUAL(info[i]->reduce_size(), spec_list[i].reduce_size); + EXPECT_EQUAL(info[i]->inner_size(), spec_list[i].inner_size); + EXPECT_EQUAL(int(info[i]->aggr()), int(spec_list[i].aggr)); + } +} + +void verify_optimized(const vespalib::string &expr, const ReduceSpec &spec) { + verify_optimized_impl(expr, {spec}); +} + +void verify_optimized(const vespalib::string &expr, const ReduceSpec &spec1, const ReduceSpec &spec2) { + verify_optimized_impl(expr, {spec1, spec2}); } void verify_not_optimized(const vespalib::string &expr) { @@ -58,11 +77,6 @@ void verify_not_optimized(const vespalib::string &expr) { EXPECT_TRUE(info.empty()); } -TEST("require that multi-dimensional reduce is not optimized") { - TEST_DO(verify_not_optimized("reduce(a2b3c4d5,sum,a,b)")); - TEST_DO(verify_not_optimized("reduce(a2b3c4d5,sum,c,d)")); -} - TEST("require that reduce to scalar is not optimized") { TEST_DO(verify_not_optimized("reduce(a10,sum,a)")); TEST_DO(verify_not_optimized("reduce(a10,sum)")); @@ -79,45 +93,83 @@ TEST("require that mixed reduce is not optimized") { TEST_DO(verify_not_optimized("reduce(xyz_mixed,sum,z)")); } -// NB: these are shadowed by the remove dimension optimization -TEST("require that reducing self-aggregating trivial dimensions is not optimized") { +TEST("require that reducing trivial dimensions is not optimized") { TEST_DO(verify_not_optimized("reduce(a1b1c1,avg,c)")); + TEST_DO(verify_not_optimized("reduce(a1b1c1,count,c)")); TEST_DO(verify_not_optimized("reduce(a1b1c1,prod,c)")); TEST_DO(verify_not_optimized("reduce(a1b1c1,sum,c)")); TEST_DO(verify_not_optimized("reduce(a1b1c1,max,c)")); + TEST_DO(verify_not_optimized("reduce(a1b1c1,median,c)")); TEST_DO(verify_not_optimized("reduce(a1b1c1,min,c)")); } -TEST("require that reducing trivial dimension with COUNT is 'optimized'") { - TEST_DO(verify_optimized("reduce(a1b1c1,count,a)", 0, Aggr::COUNT)); - TEST_DO(verify_optimized("reduce(a1b1c1,count,b)", 1, Aggr::COUNT)); - TEST_DO(verify_optimized("reduce(a1b1c1,count,c)", 2, Aggr::COUNT)); +TEST("require that atleast_8 dense single reduce works") { + TEST_DO(verify_optimized("reduce(a9b9c9d9,avg,a)", {1, 9, 729, Aggr::AVG})); + TEST_DO(verify_optimized("reduce(a9b9c9d9,avg,b)", {9, 9, 81, Aggr::AVG})); + TEST_DO(verify_optimized("reduce(a9b9c9d9,avg,c)", {81, 9, 9, Aggr::AVG})); + TEST_DO(verify_optimized("reduce(a9b9c9d9,avg,d)", {729, 9, 1, Aggr::AVG})); + TEST_DO(verify_optimized("reduce(a9b9c9d9,sum,c,d)", {81, 81, 1, Aggr::SUM})); +} + +TEST("require that simple aggregators can be decomposed into multiple reduce operations") { + TEST_DO(verify_optimized("reduce(a2b3c4d5,sum,a,c)", {3, 4, 5, Aggr::SUM}, {1, 2, 60, Aggr::SUM})); + TEST_DO(verify_optimized("reduce(a2b3c4d5,min,a,c)", {3, 4, 5, Aggr::MIN}, {1, 2, 60, Aggr::MIN})); + TEST_DO(verify_optimized("reduce(a2b3c4d5,max,a,c)", {3, 4, 5, Aggr::MAX}, {1, 2, 60, Aggr::MAX})); +} + +TEST("require that reduce dimensions can be listed in reverse order") { + TEST_DO(verify_optimized("reduce(a2b3c4d5,sum,c,a)", {3, 4, 5, Aggr::SUM}, {1, 2, 60, Aggr::SUM})); + TEST_DO(verify_optimized("reduce(a2b3c4d5,min,c,a)", {3, 4, 5, Aggr::MIN}, {1, 2, 60, Aggr::MIN})); + TEST_DO(verify_optimized("reduce(a2b3c4d5,max,c,a)", {3, 4, 5, Aggr::MAX}, {1, 2, 60, Aggr::MAX})); +} + +TEST("require that non-simple aggregators cannot be decomposed into multiple reduce operations") { + TEST_DO(verify_not_optimized("reduce(a2b3c4d5,avg,a,c)")); + TEST_DO(verify_not_optimized("reduce(a2b3c4d5,count,a,c)")); + TEST_DO(verify_not_optimized("reduce(a2b3c4d5,median,a,c)")); } vespalib::string make_expr(const vespalib::string &arg, const vespalib::string &dim, bool float_cells, Aggr aggr) { return make_string("reduce(%s%s,%s,%s)", arg.c_str(), float_cells ? "f" : "", AggrNames::name_of(aggr)->c_str(), dim.c_str()); } -void verify_optimized_multi(const vespalib::string &arg, const vespalib::string &dim, size_t dim_idx) { +void verify_optimized_multi(const vespalib::string &arg, const vespalib::string &dim, size_t outer_size, size_t reduce_size, size_t inner_size) { for (bool float_cells: {false, true}) { for (Aggr aggr: Aggregator::list()) { - auto expr = make_expr(arg, dim, float_cells, aggr); - TEST_DO(verify_optimized(expr, dim_idx, aggr)); + if (aggr != Aggr::PROD) { + auto expr = make_expr(arg, dim, float_cells, aggr); + TEST_DO(verify_optimized(expr, {outer_size, reduce_size, inner_size, aggr})); + } } } } TEST("require that normal dense single reduce works") { - TEST_DO(verify_optimized_multi("a2b3c4d5", "a", 0)); - TEST_DO(verify_optimized_multi("a2b3c4d5", "b", 1)); - TEST_DO(verify_optimized_multi("a2b3c4d5", "c", 2)); - TEST_DO(verify_optimized_multi("a2b3c4d5", "d", 3)); + TEST_DO(verify_optimized_multi("a2b3c4d5", "a", 1, 2, 60)); + TEST_DO(verify_optimized_multi("a2b3c4d5", "b", 2, 3, 20)); + TEST_DO(verify_optimized_multi("a2b3c4d5", "c", 6, 4, 5)); + TEST_DO(verify_optimized_multi("a2b3c4d5", "d", 24, 5, 1)); +} + +TEST("require that dimension-combined dense single reduce works") { + TEST_DO(verify_optimized_multi("a2b3c4d5", "a,b", 1, 6, 20)); + TEST_DO(verify_optimized_multi("a2b3c4d5", "b,c", 2, 12, 5)); + TEST_DO(verify_optimized_multi("a2b3c4d5", "c,d", 6, 20, 1)); } TEST("require that minimal dense single reduce works") { - TEST_DO(verify_optimized_multi("a2b1c1", "a", 0)); - TEST_DO(verify_optimized_multi("a1b2c1", "b", 1)); - TEST_DO(verify_optimized_multi("a1b1c2", "c", 2)); + TEST_DO(verify_optimized_multi("a2b1c1", "a", 1, 2, 1)); + TEST_DO(verify_optimized_multi("a1b2c1", "b", 1, 2, 1)); + TEST_DO(verify_optimized_multi("a1b1c2", "c", 1, 2, 1)); +} + +TEST("require that trivial dimensions can be trivially reduced") { + TEST_DO(verify_optimized_multi("a2b1c1", "a,b", 1, 2, 1)); + TEST_DO(verify_optimized_multi("a2b1c1", "a,c", 1, 2, 1)); + TEST_DO(verify_optimized_multi("a1b2c1", "b,a", 1, 2, 1)); + TEST_DO(verify_optimized_multi("a1b2c1", "b,c", 1, 2, 1)); + TEST_DO(verify_optimized_multi("a1b1c2", "c,a", 1, 2, 1)); + TEST_DO(verify_optimized_multi("a1b1c2", "c,b", 1, 2, 1)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp b/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp index d9d4c221164..bcee6471f76 100644 --- a/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp +++ b/eval/src/tests/tensor/direct_sparse_tensor_builder/direct_sparse_tensor_builder_test.cpp @@ -8,6 +8,7 @@ using namespace vespalib::tensor; using namespace vespalib::tensor::sparse; using vespalib::eval::TensorSpec; +using vespalib::eval::CellType; using vespalib::eval::ValueType; void @@ -36,7 +37,7 @@ assertCellValue(double expValue, const TensorAddress &address, bool found = tensor.index().lookup_address(addressRef, idx); EXPECT_TRUE(found); auto cells = tensor.cells(); - if (EXPECT_TRUE(cells.type == ValueType::CellType::DOUBLE)) { + if (EXPECT_TRUE(cells.type == CellType::DOUBLE)) { auto arr = cells.typify<double>(); EXPECT_EQUAL(expValue, arr[idx]); } diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp index 8887f2cb6aa..e4c1af3100a 100644 --- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -95,7 +95,7 @@ template <typename ...Ds> void add_cells(TensorSpec &spec, double &seq, TensorSp } template <typename ...Ds> TensorSpec make_spec(double seq, const Ds &...ds) { - TensorSpec spec(ValueType::tensor_type({ds...}, ValueType::CellType::FLOAT).to_spec()); + TensorSpec spec(ValueType::tensor_type({ds...}, CellType::FLOAT).to_spec()); add_cells(spec, seq, TensorSpec::Address(), ds...); return spec; } @@ -122,6 +122,32 @@ MyPeekSpec verbatim_peek() { return MyPeekSpec(false); } //----------------------------------------------------------------------------- +struct MultiOpParam { + std::vector<Instruction> list; +}; + +void my_multi_instruction_op(InterpretedFunction::State &state, uint64_t param_in) { + const auto ¶m = *(MultiOpParam*)(param_in); + for (const auto &item: param.list) { + item.perform(state); + } +} + +void collect_op1_chain(const TensorFunction &node, const EngineOrFactory &engine, Stash &stash, std::vector<Instruction> &list) { + if (auto op1 = as<tensor_function::Op1>(node)) { + collect_op1_chain(op1->child(), engine, stash, list); + list.push_back(node.compile_self(engine, stash)); + } +} + +Instruction compile_op1_chain(const TensorFunction &node, const EngineOrFactory &engine, Stash &stash) { + auto ¶m = stash.create<MultiOpParam>(); + collect_op1_chain(node, engine, stash, param.list); + return {my_multi_instruction_op,(uint64_t)(¶m)}; +} + +//----------------------------------------------------------------------------- + struct Impl { size_t order; vespalib::string name; @@ -145,7 +171,10 @@ struct Impl { const auto &lhs_node = tensor_function::inject(lhs, 0, stash); const auto &reduce_node = tensor_function::reduce(lhs_node, aggr, dims, stash); const auto &node = optimize ? optimize_tensor_function(engine, reduce_node, stash) : reduce_node; - return node.compile_self(engine, stash); + // since reduce might be optimized into multiple chained + // instructions, we need some extra magic to package these + // instructions into a single compound instruction. + return compile_op1_chain(node, engine, stash); } Instruction create_rename(const ValueType &lhs, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash) const { // create a complete tensor function, but only compile the relevant instruction @@ -325,6 +354,7 @@ MyParam::~MyParam() = default; struct EvalOp { using UP = std::unique_ptr<EvalOp>; + Stash my_stash; const Impl &impl; MyParam my_param; std::vector<Value::UP> values; @@ -332,8 +362,8 @@ struct EvalOp { EvalSingle single; EvalOp(const EvalOp &) = delete; EvalOp &operator=(const EvalOp &) = delete; - EvalOp(Instruction op, const std::vector<CREF<TensorSpec>> &stack_spec, const Impl &impl_in) - : impl(impl_in), my_param(), values(), stack(), single(impl.engine, op) + EvalOp(Stash &&stash_in, Instruction op, const std::vector<CREF<TensorSpec>> &stack_spec, const Impl &impl_in) + : my_stash(std::move(stash_in)), impl(impl_in), my_param(), values(), stack(), single(impl.engine, op) { for (const TensorSpec &spec: stack_spec) { values.push_back(impl.create_value(spec)); @@ -342,14 +372,51 @@ struct EvalOp { stack.push_back(*value.get()); } } - EvalOp(Instruction op, const TensorSpec &p0, const Impl &impl_in) - : impl(impl_in), my_param(p0, impl), values(), stack(), single(impl.engine, op, my_param) + EvalOp(Stash &&stash_in, Instruction op, const TensorSpec &p0, const Impl &impl_in) + : my_stash(std::move(stash_in)), impl(impl_in), my_param(p0, impl), values(), stack(), single(impl.engine, op, my_param) { } TensorSpec result() { return impl.create_spec(single.eval(stack)); } - double estimate_cost_us() { - auto actual = [&](){ single.eval(stack); }; - return BenchmarkTimer::benchmark(actual, budget) * 1000.0 * 1000.0; + size_t suggest_loop_cnt() { + size_t loop_cnt = 1; + auto my_loop = [&](){ + for (size_t i = 0; i < loop_cnt; ++i) { + single.eval(stack); + } + }; + for (;;) { + vespalib::BenchmarkTimer timer(0.0); + for (size_t i = 0; i < 5; ++i) { + timer.before(); + my_loop(); + timer.after(); + } + double min_time = timer.min_time(); + if (min_time > 0.004) { + break; + } else { + loop_cnt *= 2; + } + } + return std::max(loop_cnt, size_t(8)); + } + double estimate_cost_us(size_t self_loop_cnt, size_t ref_loop_cnt) { + size_t loop_cnt = ((self_loop_cnt * 128) < ref_loop_cnt) ? self_loop_cnt : ref_loop_cnt; + assert((loop_cnt % 8) == 0); + auto my_loop = [&](){ + for (size_t i = 0; (i + 7) < loop_cnt; i += 8) { + for (size_t j = 0; j < 8; ++j) { + single.eval(stack); + } + } + }; + BenchmarkTimer timer(budget); + while (timer.has_budget()) { + timer.before(); + my_loop(); + timer.after(); + } + return timer.min_time() * 1000.0 * 1000.0 / double(loop_cnt); } }; @@ -367,8 +434,12 @@ void benchmark(const vespalib::string &desc, const std::vector<EvalOp::UP> &list } } BenchmarkResult result(desc, list.size()); + std::vector<size_t> loop_cnt(list.size()); for (const auto &eval: list) { - double time = eval->estimate_cost_us(); + loop_cnt[eval->impl.order] = eval->suggest_loop_cnt(); + } + for (const auto &eval: list) { + double time = eval->estimate_cost_us(loop_cnt[eval->impl.order], loop_cnt[1]); result.sample(eval->impl.order, time); fprintf(stderr, " %s(%s): %10.3f us\n", eval->impl.name.c_str(), eval->impl.short_name.c_str(), time); } @@ -391,9 +462,10 @@ void benchmark_join(const vespalib::string &desc, const TensorSpec &lhs, ASSERT_FALSE(res_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_join(lhs_type, rhs_type, function, stash); + Stash my_stash; + auto op = impl.create_join(lhs_type, rhs_type, function, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs, rhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -410,9 +482,10 @@ void benchmark_reduce(const vespalib::string &desc, const TensorSpec &lhs, ASSERT_FALSE(res_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_reduce(lhs_type, aggr, dims, stash); + Stash my_stash; + auto op = impl.create_reduce(lhs_type, aggr, dims, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -430,9 +503,10 @@ void benchmark_rename(const vespalib::string &desc, const TensorSpec &lhs, ASSERT_FALSE(res_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_rename(lhs_type, from, to, stash); + Stash my_stash; + auto op = impl.create_rename(lhs_type, from, to, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -451,9 +525,10 @@ void benchmark_merge(const vespalib::string &desc, const TensorSpec &lhs, ASSERT_FALSE(res_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_merge(lhs_type, rhs_type, function, stash); + Stash my_stash; + auto op = impl.create_merge(lhs_type, rhs_type, function, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs, rhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -467,9 +542,10 @@ void benchmark_map(const vespalib::string &desc, const TensorSpec &lhs, operatio ASSERT_FALSE(lhs_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_map(lhs_type, function, stash); + Stash my_stash; + auto op = impl.create_map(lhs_type, function, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -488,9 +564,10 @@ void benchmark_concat(const vespalib::string &desc, const TensorSpec &lhs, ASSERT_FALSE(res_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_concat(lhs_type, rhs_type, dimension, stash); + Stash my_stash; + auto op = impl.create_concat(lhs_type, rhs_type, dimension, my_stash); std::vector<CREF<TensorSpec>> stack_spec({lhs, rhs}); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -507,10 +584,11 @@ void benchmark_tensor_create(const vespalib::string &desc, const TensorSpec &pro } std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_tensor_create(proto_type, proto, stash); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + Stash my_stash; + auto op = impl.create_tensor_create(proto_type, proto, my_stash); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } - benchmark(desc, list); + benchmark(desc, list); } //----------------------------------------------------------------------------- @@ -521,8 +599,9 @@ void benchmark_tensor_lambda(const vespalib::string &desc, const ValueType &type ASSERT_FALSE(p0_type.is_error()); std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_tensor_lambda(type, function, p0_type, stash); - list.push_back(std::make_unique<EvalOp>(op, p0, impl)); + Stash my_stash; + auto op = impl.create_tensor_lambda(type, function, p0_type, my_stash); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, p0, impl)); } benchmark(desc, list); } @@ -542,8 +621,9 @@ void benchmark_tensor_peek(const vespalib::string &desc, const TensorSpec &lhs, } std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { - auto op = impl.create_tensor_peek(type, peek_spec, stash); - list.push_back(std::make_unique<EvalOp>(op, stack_spec, impl)); + Stash my_stash; + auto op = impl.create_tensor_peek(type, peek_spec, my_stash); + list.push_back(std::make_unique<EvalOp>(std::move(my_stash), op, stack_spec, impl)); } benchmark(desc, list); } @@ -581,11 +661,11 @@ void benchmark_encode_decode(const vespalib::string &desc, const TensorSpec &pro BenchmarkResult encode_result(desc + " <encode>", impl_list.size()); BenchmarkResult decode_result(desc + " <decode>", impl_list.size()); for (const Impl &impl: impl_list) { - constexpr size_t loop_cnt = 16; + constexpr size_t loop_cnt = 32; auto value = impl.create_value(proto); BenchmarkTimer encode_timer(2 * budget); BenchmarkTimer decode_timer(2 * budget); - while (encode_timer.has_budget() || decode_timer.has_budget()) { + while (encode_timer.has_budget()) { std::array<vespalib::nbostream, loop_cnt> data; std::array<Value::UP, loop_cnt> object; encode_timer.before(); diff --git a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp index fce7ccc6411..efab0571e62 100644 --- a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp +++ b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp @@ -163,7 +163,7 @@ TEST(OnnxTest, simple_onnx_model_can_be_evaluated) ctx.bind_param(2, bias); ctx.eval(); auto cells = output.cells(); - EXPECT_EQ(cells.type, ValueType::CellType::FLOAT); + EXPECT_EQ(cells.type, CellType::FLOAT); EXPECT_EQ(cells.size, 1); EXPECT_EQ(GetCell::from(cells, 0), 79.0); //------------------------------------------------------------------------- @@ -209,7 +209,7 @@ TEST(OnnxTest, dynamic_onnx_model_can_be_evaluated) ctx.bind_param(2, bias); ctx.eval(); auto cells = output.cells(); - EXPECT_EQ(cells.type, ValueType::CellType::FLOAT); + EXPECT_EQ(cells.type, CellType::FLOAT); EXPECT_EQ(cells.size, 1); EXPECT_EQ(GetCell::from(cells, 0), 79.0); //------------------------------------------------------------------------- @@ -255,7 +255,7 @@ TEST(OnnxTest, int_types_onnx_model_can_be_evaluated) ctx.bind_param(2, bias); ctx.eval(); auto cells = output.cells(); - EXPECT_EQ(cells.type, ValueType::CellType::DOUBLE); + EXPECT_EQ(cells.type, CellType::DOUBLE); EXPECT_EQ(cells.size, 1); EXPECT_EQ(GetCell::from(cells, 0), 79.0); //------------------------------------------------------------------------- diff --git a/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp b/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp index e182fffa890..5af2396f5ec 100644 --- a/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp +++ b/eval/src/tests/tensor/partial_remove/partial_remove_test.cpp @@ -124,20 +124,31 @@ expect_partial_remove(const TensorSpec& input, const TensorSpec& remove, const T } TEST(PartialRemoveTest, remove_where_address_is_not_fully_specified) { - auto input = TensorSpec("tensor(x{},y{})"). + auto input_sparse = TensorSpec("tensor(x{},y{})"). add({{"x", "a"},{"y", "c"}}, 3.0). add({{"x", "a"},{"y", "d"}}, 5.0). add({{"x", "b"},{"y", "c"}}, 7.0); - expect_partial_remove(input,TensorSpec("tensor(x{})").add({{"x", "a"}}, 1.0), + expect_partial_remove(input_sparse, TensorSpec("tensor(x{})").add({{"x", "a"}}, 1.0), TensorSpec("tensor(x{},y{})").add({{"x", "b"},{"y", "c"}}, 7.0)); - expect_partial_remove(input, TensorSpec("tensor(y{})").add({{"y", "c"}}, 1.0), + expect_partial_remove(input_sparse, TensorSpec("tensor(y{})").add({{"y", "c"}}, 1.0), TensorSpec("tensor(x{},y{})").add({{"x", "a"},{"y", "d"}}, 5.0)); - expect_partial_remove(input, TensorSpec("tensor(y{})").add({{"y", "d"}}, 1.0), + expect_partial_remove(input_sparse, TensorSpec("tensor(y{})").add({{"y", "d"}}, 1.0), TensorSpec("tensor(x{},y{})").add({{"x", "a"},{"y", "c"}}, 3.0) .add({{"x", "b"},{"y", "c"}}, 7.0)); + + auto input_mixed = TensorSpec("tensor(x{},y{},z[1])"). + add({{"x", "a"},{"y", "c"},{"z", 0}}, 3.0). + add({{"x", "a"},{"y", "d"},{"z", 0}}, 5.0). + add({{"x", "b"},{"y", "c"},{"z", 0}}, 7.0); + + expect_partial_remove(input_mixed,TensorSpec("tensor(x{})").add({{"x", "a"}}, 1.0), + TensorSpec("tensor(x{},y{},z[1])").add({{"x", "b"},{"y", "c"},{"z", 0}}, 7.0)); + + expect_partial_remove(input_mixed, TensorSpec("tensor(y{})").add({{"y", "c"}}, 1.0), + TensorSpec("tensor(x{},y{},z[1])").add({{"x", "a"},{"y", "d"},{"z", 0}}, 5.0)); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt index d27de8e3d21..5cf7440237b 100644 --- a/eval/src/vespa/eval/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(eval_eval OBJECT array_array_map.cpp basic_nodes.cpp call_nodes.cpp + cell_type.cpp compile_tensor_function.cpp delete_node.cpp double_value_builder.cpp diff --git a/eval/src/vespa/eval/eval/aggr.cpp b/eval/src/vespa/eval/eval/aggr.cpp index 4abd5e41f47..a73cc4314c6 100644 --- a/eval/src/vespa/eval/eval/aggr.cpp +++ b/eval/src/vespa/eval/eval/aggr.cpp @@ -17,6 +17,7 @@ struct Wrapper : Aggregator { virtual void first(double value) final override { aggr = T{value}; } virtual void next(double value) final override { aggr.sample(value); } virtual double result() const final override { return aggr.result(); } + virtual Aggr enum_value() const final override { return T::enum_value(); } }; } // namespace vespalib::eval::<unnamed> diff --git a/eval/src/vespa/eval/eval/aggr.h b/eval/src/vespa/eval/eval/aggr.h index f52c029eee5..e69b1071e61 100644 --- a/eval/src/vespa/eval/eval/aggr.h +++ b/eval/src/vespa/eval/eval/aggr.h @@ -53,6 +53,7 @@ struct Aggregator { virtual void first(double value) = 0; virtual void next(double value) = 0; virtual double result() const = 0; + virtual Aggr enum_value() const = 0; virtual ~Aggregator(); static Aggregator &create(Aggr aggr, Stash &stash); static std::vector<Aggr> list(); @@ -60,11 +61,37 @@ struct Aggregator { namespace aggr { +// can we start by picking any value from the set to be reduced and +// use the templated aggregator 'combine' function in arbitrary order +// to end up with (approximately) the correct result? +constexpr bool is_simple(Aggr aggr) { + return ((aggr == Aggr::PROD) || + (aggr == Aggr::SUM) || + (aggr == Aggr::MAX) || + (aggr == Aggr::MIN)); +} + +// will a single value reduce to itself? +constexpr bool is_ident(Aggr aggr) { + return ((aggr == Aggr::AVG) || + (aggr == Aggr::PROD) || + (aggr == Aggr::SUM) || + (aggr == Aggr::MAX) || + (aggr == Aggr::MEDIAN) || + (aggr == Aggr::MIN)); +} + +// should we avoid doing clever stuff with this aggregator? +constexpr bool is_complex(Aggr aggr) { + return (aggr == Aggr::MEDIAN); +} + template <typename T> class Avg { private: T _sum; size_t _cnt; public: + using value_type = T; constexpr Avg() : _sum{0}, _cnt{0} {} constexpr Avg(T value) : _sum{value}, _cnt{1} {} constexpr void sample(T value) { @@ -76,56 +103,69 @@ public: _cnt += rhs._cnt; }; constexpr T result() const { return (_sum / _cnt); } + static constexpr Aggr enum_value() { return Aggr::AVG; } }; template <typename T> class Count { private: size_t _cnt; public: + using value_type = T; constexpr Count() : _cnt{0} {} constexpr Count(T) : _cnt{1} {} constexpr void sample(T) { ++_cnt; } constexpr void merge(const Count &rhs) { _cnt += rhs._cnt; } constexpr T result() const { return _cnt; } + static constexpr Aggr enum_value() { return Aggr::COUNT; } }; template <typename T> class Prod { private: T _prod; public: + using value_type = T; constexpr Prod() : _prod{1} {} constexpr Prod(T value) : _prod{value} {} constexpr void sample(T value) { _prod *= value; } constexpr void merge(const Prod &rhs) { _prod *= rhs._prod; } constexpr T result() const { return _prod; } + static constexpr Aggr enum_value() { return Aggr::PROD; } + static constexpr T combine(T a, T b) { return (a * b); } }; template <typename T> class Sum { private: T _sum; public: + using value_type = T; constexpr Sum() : _sum{0} {} constexpr Sum(T value) : _sum{value} {} constexpr void sample(T value) { _sum += value; } constexpr void merge(const Sum &rhs) { _sum += rhs._sum; } constexpr T result() const { return _sum; } + static constexpr Aggr enum_value() { return Aggr::SUM; } + static constexpr T combine(T a, T b) { return (a + b); } }; template <typename T> class Max { private: T _max; public: + using value_type = T; constexpr Max() : _max{-std::numeric_limits<T>::infinity()} {} constexpr Max(T value) : _max{value} {} constexpr void sample(T value) { _max = std::max(_max, value); } constexpr void merge(const Max &rhs) { _max = std::max(_max, rhs._max); } constexpr T result() const { return _max; } + static constexpr Aggr enum_value() { return Aggr::MAX; } + static constexpr T combine(T a, T b) { return std::max(a,b); } }; template <typename T> class Median { private: std::vector<T> _seen; public: + using value_type = T; constexpr Median() : _seen() {} constexpr Median(T value) : _seen({value}) {} constexpr void sample(T value) { _seen.push_back(value); } @@ -156,20 +196,24 @@ public: } return result; } + static constexpr Aggr enum_value() { return Aggr::MEDIAN; } }; template <typename T> class Min { private: T _min; public: + using value_type = T; constexpr Min() : _min{std::numeric_limits<T>::infinity()} {} constexpr Min(T value) : _min{value} {} constexpr void sample(T value) { _min = std::min(_min, value); } constexpr void merge(const Min &rhs) { _min = std::min(_min, rhs._min); } constexpr T result() const { return _min; } + static constexpr Aggr enum_value() { return Aggr::MIN; } + static constexpr T combine(T a, T b) { return std::min(a,b); } }; -} // namespave vespalib::eval::aggr +} // namespace vespalib::eval::aggr struct TypifyAggr { template <template<typename> typename TT> using Result = TypifyResultSimpleTemplate<TT>; diff --git a/eval/src/vespa/eval/eval/cell_type.cpp b/eval/src/vespa/eval/eval/cell_type.cpp new file mode 100644 index 00000000000..e5729c547b0 --- /dev/null +++ b/eval/src/vespa/eval/eval/cell_type.cpp @@ -0,0 +1,3 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "cell_type.h" diff --git a/eval/src/vespa/eval/eval/cell_type.h b/eval/src/vespa/eval/eval/cell_type.h new file mode 100644 index 00000000000..0e878f26f47 --- /dev/null +++ b/eval/src/vespa/eval/eval/cell_type.h @@ -0,0 +1,39 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/typify.h> +#include <cstdlib> + +namespace vespalib::eval { + +enum class CellType : char { FLOAT, DOUBLE }; + +// utility templates + +template <typename CT> inline bool check_cell_type(CellType type); +template <> inline bool check_cell_type<double>(CellType type) { return (type == CellType::DOUBLE); } +template <> inline bool check_cell_type<float>(CellType type) { return (type == CellType::FLOAT); } + +template <typename LCT, typename RCT> struct UnifyCellTypes{}; +template <> struct UnifyCellTypes<double, double> { using type = double; }; +template <> struct UnifyCellTypes<double, float> { using type = double; }; +template <> struct UnifyCellTypes<float, double> { using type = double; }; +template <> struct UnifyCellTypes<float, float> { using type = float; }; + +template <typename CT> inline CellType get_cell_type(); +template <> inline CellType get_cell_type<double>() { return CellType::DOUBLE; } +template <> inline CellType get_cell_type<float>() { return CellType::FLOAT; } + +struct TypifyCellType { + template <typename T> using Result = TypifyResultType<T>; + template <typename F> static decltype(auto) resolve(CellType value, F &&f) { + switch(value) { + case CellType::DOUBLE: return f(Result<double>()); + case CellType::FLOAT: return f(Result<float>()); + } + abort(); + } +}; + +} // namespace diff --git a/eval/src/vespa/eval/eval/fast_sparse_map.cpp b/eval/src/vespa/eval/eval/fast_sparse_map.cpp index 2e95934286c..e5ffbb5c515 100644 --- a/eval/src/vespa/eval/eval/fast_sparse_map.cpp +++ b/eval/src/vespa/eval/eval/fast_sparse_map.cpp @@ -7,6 +7,12 @@ namespace vespalib::eval { FastSparseMap::~FastSparseMap() = default; +FastSparseMap& +FastSparseMap::operator=(const FastSparseMap& rhs) = default; + +FastSparseMap& +FastSparseMap::operator=(FastSparseMap&& rhs) = default; + const FastSparseMap::HashedLabel FastSparseMap::empty_label; } diff --git a/eval/src/vespa/eval/eval/fast_sparse_map.h b/eval/src/vespa/eval/eval/fast_sparse_map.h index 0d7597a19a0..99e01e8c823 100644 --- a/eval/src/vespa/eval/eval/fast_sparse_map.h +++ b/eval/src/vespa/eval/eval/fast_sparse_map.h @@ -97,6 +97,9 @@ public: } ~FastSparseMap(); + FastSparseMap& operator=(const FastSparseMap& rhs); + FastSparseMap& operator=(FastSparseMap&& rhs); + MemoryUsage estimate_extra_memory_usage() const { MemoryUsage extra_usage; size_t map_self_size = sizeof(_map); diff --git a/eval/src/vespa/eval/eval/fast_value.hpp b/eval/src/vespa/eval/eval/fast_value.hpp index ff94f94efbc..9914378cc9e 100644 --- a/eval/src/vespa/eval/eval/fast_value.hpp +++ b/eval/src/vespa/eval/eval/fast_value.hpp @@ -390,20 +390,12 @@ FastValueIndex::sparse_only_merge(const ValueType &res_type, const Fun &fun, const FastValueIndex &lhs, const FastValueIndex &rhs, ConstArrayRef<LCT> lhs_cells, ConstArrayRef<RCT> rhs_cells, Stash &stash) { - auto &result = stash.create<FastValue<OCT>>(res_type, lhs.map.num_dims(), 1, lhs.map.size()+rhs.map.size()); - lhs.map.each_map_entry([&](auto lhs_subspace, auto hash) - { - auto idx = result.my_index.map.add_mapping(lhs.map.make_addr(lhs_subspace), hash); - if (__builtin_expect((idx == result.my_cells.size), true)) { - auto rhs_subspace = rhs.map.lookup(hash); - if (rhs_subspace != FastSparseMap::npos()) { - auto cell_value = fun(lhs_cells[lhs_subspace], rhs_cells[rhs_subspace]); - result.my_cells.push_back_fast(cell_value); - } else { - result.my_cells.push_back_fast(lhs_cells[lhs_subspace]); - } - } - }); + size_t guess_size = lhs.map.size() + rhs.map.size(); + auto &result = stash.create<FastValue<OCT>>(res_type, lhs.map.num_dims(), 1, guess_size); + result.my_index = lhs; + for (auto val : lhs_cells) { + result.my_cells.push_back_fast(val); + } rhs.map.each_map_entry([&](auto rhs_subspace, auto hash) { auto lhs_subspace = lhs.map.lookup(hash); @@ -412,9 +404,11 @@ FastValueIndex::sparse_only_merge(const ValueType &res_type, const Fun &fun, if (__builtin_expect((idx == result.my_cells.size), true)) { result.my_cells.push_back_fast(rhs_cells[rhs_subspace]); } + } else { + auto cell_value = fun(lhs_cells[lhs_subspace], rhs_cells[rhs_subspace]); + *result.my_cells.get(lhs_subspace) = cell_value; } }); - return result; } diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp index 64b2b6f8865..98e3bc325cb 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor.cpp @@ -18,7 +18,6 @@ using Cells = SimpleTensor::Cells; using IndexList = std::vector<size_t>; using Label = SimpleTensor::Label; using CellRef = std::reference_wrapper<const Cell>; -using CellType = ValueType::CellType; namespace { diff --git a/eval/src/vespa/eval/eval/simple_value.cpp b/eval/src/vespa/eval/eval/simple_value.cpp index 766a4f1eb23..17faa635941 100644 --- a/eval/src/vespa/eval/eval/simple_value.cpp +++ b/eval/src/vespa/eval/eval/simple_value.cpp @@ -3,8 +3,6 @@ #include "simple_value.h" #include "inline_operation.h" #include <vespa/vespalib/util/typify.h> -#include <vespa/vespalib/util/visit_ranges.h> -#include <vespa/vespalib/util/overload.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> diff --git a/eval/src/vespa/eval/eval/test/tensor_model.hpp b/eval/src/vespa/eval/eval/test/tensor_model.hpp index 59653954c9e..78d6798ac4c 100644 --- a/eval/src/vespa/eval/eval/test/tensor_model.hpp +++ b/eval/src/vespa/eval/eval/test/tensor_model.hpp @@ -16,7 +16,6 @@ namespace vespalib { namespace eval { namespace test { -using CellType = ValueType::CellType; using map_fun_t = vespalib::eval::operation::op1_t; using join_fun_t = vespalib::eval::operation::op2_t; diff --git a/eval/src/vespa/eval/eval/typed_cells.h b/eval/src/vespa/eval/eval/typed_cells.h index 09d5c080cf7..a478a419f95 100644 --- a/eval/src/vespa/eval/eval/typed_cells.h +++ b/eval/src/vespa/eval/eval/typed_cells.h @@ -11,8 +11,6 @@ namespace vespalib::eval { // Low-level typed cells reference struct TypedCells { - using CellType = vespalib::eval::ValueType::CellType; - const void *data; CellType type; size_t size:56; diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index 2de95657f72..923d3f29cd3 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -14,8 +14,6 @@ namespace vespalib::eval { namespace { -using CellType = ValueType::CellType; - constexpr uint32_t DOUBLE_CELL_TYPE = 0; constexpr uint32_t FLOAT_CELL_TYPE = 1; @@ -118,7 +116,7 @@ ValueType decode_type(nbostream &input, const Format &format) { } } if (dim_list.empty()) { - assert(cell_type == ValueType::CellType::DOUBLE); + assert(cell_type == CellType::DOUBLE); } return ValueType::tensor_type(std::move(dim_list), cell_type); } diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index c7d77c766bc..05ec65bf292 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -8,7 +8,6 @@ namespace vespalib::eval { namespace { -using CellType = ValueType::CellType; using Dimension = ValueType::Dimension; using DimensionList = std::vector<Dimension>; diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index 38f4705edf2..6d9316e76ed 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/util/typify.h> +#include "cell_type.h" #include <vespa/vespalib/stllike/string.h> #include <vector> @@ -16,7 +16,6 @@ namespace vespalib::eval { class ValueType { public: - enum class CellType : char { FLOAT, DOUBLE }; struct Dimension { using size_type = uint32_t; static constexpr size_type npos = -1; @@ -110,31 +109,4 @@ public: std::ostream &operator<<(std::ostream &os, const ValueType &type); -// utility templates - -template <typename CT> inline bool check_cell_type(ValueType::CellType type); -template <> inline bool check_cell_type<double>(ValueType::CellType type) { return (type == ValueType::CellType::DOUBLE); } -template <> inline bool check_cell_type<float>(ValueType::CellType type) { return (type == ValueType::CellType::FLOAT); } - -template <typename LCT, typename RCT> struct UnifyCellTypes{}; -template <> struct UnifyCellTypes<double, double> { using type = double; }; -template <> struct UnifyCellTypes<double, float> { using type = double; }; -template <> struct UnifyCellTypes<float, double> { using type = double; }; -template <> struct UnifyCellTypes<float, float> { using type = float; }; - -template <typename CT> inline ValueType::CellType get_cell_type(); -template <> inline ValueType::CellType get_cell_type<double>() { return ValueType::CellType::DOUBLE; } -template <> inline ValueType::CellType get_cell_type<float>() { return ValueType::CellType::FLOAT; } - -struct TypifyCellType { - template <typename T> using Result = TypifyResultType<T>; - template <typename F> static decltype(auto) resolve(ValueType::CellType value, F &&f) { - switch(value) { - case ValueType::CellType::DOUBLE: return f(Result<double>()); - case ValueType::CellType::FLOAT: return f(Result<float>()); - } - abort(); - } -}; - } // namespace diff --git a/eval/src/vespa/eval/eval/value_type_spec.cpp b/eval/src/vespa/eval/eval/value_type_spec.cpp index 847203db3b1..a4575e33c2f 100644 --- a/eval/src/vespa/eval/eval/value_type_spec.cpp +++ b/eval/src/vespa/eval/eval/value_type_spec.cpp @@ -8,8 +8,6 @@ namespace vespalib::eval::value_type { -using CellType = ValueType::CellType; - namespace { const char *to_name(CellType cell_type) { @@ -188,7 +186,7 @@ parse_spec(const char *pos_in, const char *end_in, const char *&pos_out, } else if (type_name == "float") { return ValueType::make_type(CellType::FLOAT, {}); } else if (type_name == "tensor") { - ValueType::CellType cell_type = parse_cell_type(ctx); + CellType cell_type = parse_cell_type(ctx); std::vector<ValueType::Dimension> list = parse_dimension_list(ctx); if (!ctx.failed()) { if (unsorted != nullptr) { diff --git a/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp b/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp index cc746c4db83..5dcfcba025d 100644 --- a/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_dot_product_function.cpp @@ -45,12 +45,12 @@ struct MyDotProductOp { static auto invoke() { return my_dot_product_op<LCT,RCT>; } }; -InterpretedFunction::op_function my_select(ValueType::CellType lct, ValueType::CellType rct) { +InterpretedFunction::op_function my_select(CellType lct, CellType rct) { if (lct == rct) { - if (lct == ValueType::CellType::DOUBLE) { + if (lct == CellType::DOUBLE) { return my_cblas_double_dot_product_op; } - if (lct == ValueType::CellType::FLOAT) { + if (lct == CellType::FLOAT) { return my_cblas_float_dot_product_op; } } diff --git a/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp b/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp index cbca2ff14f2..42e7deb9523 100644 --- a/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp +++ b/eval/src/vespa/eval/instruction/dense_multi_matmul_function.cpp @@ -60,11 +60,11 @@ void my_cblas_float_multi_matmul_op(InterpretedFunction::State &state, uint64_t state.pop_pop_push(state.stash.create<tensor::DenseTensorView>(self.result_type(), TypedCells(dst_cells))); } -InterpretedFunction::op_function my_select(ValueType::CellType cell_type) { - if (cell_type == ValueType::CellType::DOUBLE) { +InterpretedFunction::op_function my_select(CellType cell_type) { + if (cell_type == CellType::DOUBLE) { return my_cblas_double_multi_matmul_op; } - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return my_cblas_float_multi_matmul_op; } abort(); @@ -117,7 +117,7 @@ struct DimPrefix { bool check_input_type(const ValueType &type, const DimList &relevant) { return (type.is_dense() && (relevant.size() >= 2) && - ((type.cell_type() == ValueType::CellType::FLOAT) || (type.cell_type() == ValueType::CellType::DOUBLE))); + ((type.cell_type() == CellType::FLOAT) || (type.cell_type() == CellType::DOUBLE))); } bool is_multi_matmul(const ValueType &a, const ValueType &b, const vespalib::string &reduce_dim) { diff --git a/eval/src/vespa/eval/instruction/generic_merge.cpp b/eval/src/vespa/eval/instruction/generic_merge.cpp index 87be47a9c2e..8de4ea1adeb 100644 --- a/eval/src/vespa/eval/instruction/generic_merge.cpp +++ b/eval/src/vespa/eval/instruction/generic_merge.cpp @@ -127,9 +127,19 @@ void my_sparse_merge_op(State &state, uint64_t param_in) { if (auto indexes = detect_type<FastValueIndex>(lhs.index(), rhs.index())) { auto lhs_cells = lhs.cells().typify<LCT>(); auto rhs_cells = rhs.cells().typify<RCT>(); - return state.pop_pop_push( + if (lhs_cells.size() < rhs_cells.size()) { + return state.pop_pop_push( + FastValueIndex::sparse_only_merge<RCT,LCT,OCT,Fun>( + param.res_type, Fun(param.function), + indexes.get<1>(), indexes.get<0>(), + rhs_cells, lhs_cells, state.stash)); + } else { + return state.pop_pop_push( FastValueIndex::sparse_only_merge<LCT,RCT,OCT,Fun>( - param.res_type, Fun(param.function), indexes.get<0>(), indexes.get<1>(), lhs_cells, rhs_cells, state.stash)); + param.res_type, Fun(param.function), + indexes.get<0>(), indexes.get<1>(), + lhs_cells, rhs_cells, state.stash)); + } } auto up = generic_mixed_merge<LCT, RCT, OCT, Fun>(lhs, rhs, param); auto &result = state.stash.create<std::unique_ptr<Value>>(std::move(up)); diff --git a/eval/src/vespa/eval/instruction/generic_peek.cpp b/eval/src/vespa/eval/instruction/generic_peek.cpp index 651ce4df28a..5802a60d43a 100644 --- a/eval/src/vespa/eval/instruction/generic_peek.cpp +++ b/eval/src/vespa/eval/instruction/generic_peek.cpp @@ -46,12 +46,12 @@ struct DimSpec { return std::get<size_t>(child_or_label); } vespalib::stringref get_label_name() const { - auto label = std::get<TensorSpec::Label>(child_or_label); + auto & label = std::get<TensorSpec::Label>(child_or_label); assert(label.is_mapped()); return label.name; } size_t get_label_index() const { - auto label = std::get<TensorSpec::Label>(child_or_label); + auto & label = std::get<TensorSpec::Label>(child_or_label); assert(label.is_indexed()); return label.index; } diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp index 04aad776e43..69177e690e4 100644 --- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp +++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp @@ -46,7 +46,6 @@ using eval::TensorFunction; using eval::TensorSpec; using eval::Value; using eval::ValueType; -using CellType = eval::ValueType::CellType; using vespalib::IllegalArgumentException; using vespalib::make_string; diff --git a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp index d6995256411..4869270532f 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_number_join_function.cpp @@ -11,6 +11,7 @@ namespace vespalib::tensor { using vespalib::ArrayRef; +using eval::CellType; using eval::Value; using eval::ValueType; using eval::TensorFunction; @@ -66,7 +67,7 @@ using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool>; bool is_dense(const TensorFunction &tf) { return tf.result_type().is_dense(); } bool is_double(const TensorFunction &tf) { return tf.result_type().is_double(); } -ValueType::CellType cell_type(const TensorFunction &tf) { return tf.result_type().cell_type(); } +CellType cell_type(const TensorFunction &tf) { return tf.result_type().cell_type(); } } // namespace vespalib::tensor::<unnamed> diff --git a/eval/src/vespa/eval/tensor/dense/dense_remove_dimension_optimizer.cpp b/eval/src/vespa/eval/tensor/dense/dense_remove_dimension_optimizer.cpp index 0cecd588317..a48527e83f5 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_remove_dimension_optimizer.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_remove_dimension_optimizer.cpp @@ -14,15 +14,6 @@ using namespace eval::tensor_function; namespace { -bool is_ident_aggr(Aggr aggr) { - return ((aggr == Aggr::AVG) || - (aggr == Aggr::PROD) || - (aggr == Aggr::SUM) || - (aggr == Aggr::MAX) || - (aggr == Aggr::MEDIAN) || - (aggr == Aggr::MIN)); -} - bool is_trivial_dim_list(const ValueType &type, const std::vector<vespalib::string> &dim_list) { size_t npos = ValueType::Dimension::npos; for (const vespalib::string &dim: dim_list) { @@ -43,7 +34,7 @@ DenseRemoveDimensionOptimizer::optimize(const eval::TensorFunction &expr, Stash const TensorFunction &child = reduce->child(); if (expr.result_type().is_dense() && child.result_type().is_dense() && - is_ident_aggr(reduce->aggr()) && + eval::aggr::is_ident(reduce->aggr()) && is_trivial_dim_list(child.result_type(), reduce->dimensions())) { assert(expr.result_type().cell_type() == child.result_type().cell_type()); diff --git a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp index 5aca3799258..add26e4e52f 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_simple_join_function.cpp @@ -14,6 +14,7 @@ namespace vespalib::tensor { using vespalib::ArrayRef; +using eval::CellType; using eval::Value; using eval::ValueType; using eval::TensorFunction; @@ -106,11 +107,11 @@ using MyTypify = TypifyValue<TypifyCellType,TypifyOp2,TypifyBool,TypifyOverlap>; //----------------------------------------------------------------------------- -bool can_use_as_output(const TensorFunction &fun, ValueType::CellType result_cell_type) { +bool can_use_as_output(const TensorFunction &fun, CellType result_cell_type) { return (fun.result_is_mutable() && (fun.result_type().cell_type() == result_cell_type)); } -Primary select_primary(const TensorFunction &lhs, const TensorFunction &rhs, ValueType::CellType result_cell_type) { +Primary select_primary(const TensorFunction &lhs, const TensorFunction &rhs, CellType result_cell_type) { size_t lhs_size = lhs.result_type().dense_subspace_size(); size_t rhs_size = rhs.result_type().dense_subspace_size(); if (lhs_size > rhs_size) { diff --git a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp index 4ca15a3b5ac..5f688657645 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.cpp @@ -4,6 +4,7 @@ #include "dense_tensor_view.h" #include <vespa/vespalib/util/typify.h> #include <vespa/eval/eval/value.h> +#include <cassert> namespace vespalib::tensor { @@ -25,27 +26,16 @@ namespace { struct Params { const ValueType &result_type; size_t outer_size; - size_t dim_size; + size_t reduce_size; size_t inner_size; - Params(const ValueType &result_type_in, const ValueType &child_type, size_t dim_idx) - : result_type(result_type_in), outer_size(1), dim_size(1), inner_size(1) - { - for (size_t i = 0; i < child_type.dimensions().size(); ++i) { - if (i < dim_idx) { - outer_size *= child_type.dimensions()[i].size; - } else if (i == dim_idx) { - dim_size *= child_type.dimensions()[i].size; - } else { - inner_size *= child_type.dimensions()[i].size; - } - } - } + Params(const ValueType &result_type_in, size_t outer_size_in, size_t reduce_size_in, size_t inner_size_in) + : result_type(result_type_in), outer_size(outer_size_in), reduce_size(reduce_size_in), inner_size(inner_size_in) {} }; template <typename CT, typename AGGR> -CT reduce_cells(const CT *src, size_t dim_size, size_t stride) { +CT reduce_cells(const CT *src, size_t reduce_size, size_t stride) { AGGR aggr(*src); - for (size_t i = 1; i < dim_size; ++i) { + for (size_t i = 1; i < reduce_size; ++i) { src += stride; aggr.sample(*src); } @@ -86,45 +76,160 @@ auto reduce_cells_atleast_8(const CT *src, size_t n, size_t stride) { } template <typename CT, typename AGGR, bool atleast_8, bool is_inner> -void my_single_reduce_op(InterpretedFunction::State &state, uint64_t param) { - const auto ¶ms = unwrap_param<Params>(param); - const CT *src = state.peek(0).cells().typify<CT>().cbegin(); - auto dst_cells = state.stash.create_uninitialized_array<CT>(params.outer_size * params.inner_size); - CT *dst = dst_cells.begin(); - const size_t block_size = (params.dim_size * params.inner_size); +void trace_reduce_impl(const Params ¶ms, const CT *src, CT *dst) { + constexpr bool aggr_is_complex = is_complex(AGGR::enum_value()); + const size_t block_size = (params.reduce_size * params.inner_size); for (size_t outer = 0; outer < params.outer_size; ++outer) { for (size_t inner = 0; inner < params.inner_size; ++inner) { - if (atleast_8) { + if (atleast_8 && !aggr_is_complex) { if (is_inner) { - *dst++ = reduce_cells_atleast_8<CT, AGGR>(src + inner, params.dim_size); + *dst++ = reduce_cells_atleast_8<CT, AGGR>(src + inner, params.reduce_size); } else { - *dst++ = reduce_cells_atleast_8<CT, AGGR>(src + inner, params.dim_size, params.inner_size); + *dst++ = reduce_cells_atleast_8<CT, AGGR>(src + inner, params.reduce_size, params.inner_size); } } else { - *dst++ = reduce_cells<CT, AGGR>(src + inner, params.dim_size, params.inner_size); + *dst++ = reduce_cells<CT, AGGR>(src + inner, params.reduce_size, params.inner_size); } } src += block_size; } +} + +template <typename CT, typename AGGR> +void fold_reduce_impl(const Params ¶ms, const CT *src, CT *dst) { + for (size_t outer = 0; outer < params.outer_size; ++outer) { + auto saved_dst = dst; + for (size_t inner = 0; inner < params.inner_size; ++inner) { + *dst++ = *src++; + } + for (size_t dim = 1; dim < params.reduce_size; ++dim) { + dst = saved_dst; + for (size_t inner = 0; inner < params.inner_size; ++inner) { + *dst = AGGR::combine(*dst, *src++); + ++dst; + } + } + } +} + +template <typename CT, typename AGGR, bool atleast_8, bool is_inner> +void my_single_reduce_op(InterpretedFunction::State &state, uint64_t param) { + static_assert(std::is_same_v<CT,typename AGGR::value_type>); + constexpr bool aggr_is_simple = is_simple(AGGR::enum_value()); + const auto ¶ms = unwrap_param<Params>(param); + const CT *src = state.peek(0).cells().typify<CT>().cbegin(); + auto dst_cells = state.stash.create_uninitialized_array<CT>(params.outer_size * params.inner_size); + CT *dst = dst_cells.begin(); + if constexpr (aggr_is_simple && !is_inner) { + fold_reduce_impl<CT, AGGR>(params, src, dst); + } else { + trace_reduce_impl<CT,AGGR,atleast_8,is_inner>(params, src, dst); + } state.pop_push(state.stash.create<DenseTensorView>(params.result_type, TypedCells(dst_cells))); } struct MyGetFun { template <typename R1, typename R2, typename R3, typename R4> static auto invoke() { - return my_single_reduce_op<R1, typename R2::template templ<R1>, R3::value, R4::value>; + using AggrType = typename R2::template templ<R1>; + return my_single_reduce_op<R1, AggrType, R3::value, R4::value>; } }; using MyTypify = TypifyValue<TypifyCellType,TypifyAggr,TypifyBool>; +std::pair<std::vector<vespalib::string>,ValueType> sort_and_drop_trivial(const std::vector<vespalib::string> &list_in, const ValueType &type_in) { + std::vector<vespalib::string> dropped; + std::vector<vespalib::string> list_out; + for (const auto &dim_name: list_in) { + auto dim_idx = type_in.dimension_index(dim_name); + assert(dim_idx != ValueType::Dimension::npos); + const auto &dim = type_in.dimensions()[dim_idx]; + assert(dim.is_indexed()); + if (dim.is_trivial()) { + dropped.push_back(dim_name); + } else { + list_out.push_back(dim_name); + } + } + std::sort(list_out.begin(), list_out.end()); + ValueType type_out = dropped.empty() ? type_in : type_in.reduce(dropped); + assert(!type_out.is_error()); + return {list_out, type_out}; +} + +template <typename T> struct VectorLookupLoop { + const std::vector<T> &list; + size_t index; + VectorLookupLoop(const std::vector<T> &list_in) : list(list_in), index(0) {} + bool valid() const { return (index < list.size()); } + void next() { ++index; } + const T &get() const { return list[index]; } +}; + +DenseSingleReduceSpec extract_next(const eval::ValueType &type, eval::Aggr aggr, + std::vector<vespalib::string> &todo) +{ + size_t outer_size = 1; + size_t reduce_size = 1; + size_t inner_size = 1; + auto dims = type.nontrivial_indexed_dimensions(); + std::vector<vespalib::string> do_now; + std::vector<vespalib::string> do_later; + auto a = VectorLookupLoop(dims); + auto b = VectorLookupLoop(todo); + while (a.valid() && b.valid() && (a.get().name < b.get())) { + outer_size *= a.get().size; + a.next(); + } + while (a.valid() && b.valid() && (a.get().name == b.get())) { + reduce_size *= a.get().size; + do_now.push_back(b.get()); + a.next(); + b.next(); + } + while (a.valid()) { + inner_size *= a.get().size; + a.next(); + } + while (b.valid()) { + do_later.push_back(b.get()); + b.next(); + } + todo = do_later; + assert(!do_now.empty()); + return {type.reduce(do_now), outer_size, reduce_size, inner_size, aggr}; +} + } // namespace vespalib::tensor::<unnamed> -DenseSingleReduceFunction::DenseSingleReduceFunction(const ValueType &result_type, - const TensorFunction &child, - size_t dim_idx, Aggr aggr) - : Op1(result_type, child), - _dim_idx(dim_idx), - _aggr(aggr) +std::vector<DenseSingleReduceSpec> +make_dense_single_reduce_list(const eval::ValueType &type, eval::Aggr aggr, + const std::vector<vespalib::string> &reduce_dims) +{ + auto res_type = type.reduce(reduce_dims); + if (reduce_dims.empty() || !type.is_dense() || !res_type.is_dense()) { + return {}; + } + std::vector<DenseSingleReduceSpec> list; + auto [todo, curr_type] = sort_and_drop_trivial(reduce_dims, type); + while (!todo.empty()) { + list.push_back(extract_next(curr_type, aggr, todo)); + curr_type = list.back().result_type; + } + assert(curr_type == res_type); + if ((list.size() > 1) && !eval::aggr::is_simple(aggr)) { + return {}; + } + return list; +} + +DenseSingleReduceFunction::DenseSingleReduceFunction(const DenseSingleReduceSpec &spec, + const TensorFunction &child) + : Op1(spec.result_type, child), + _outer_size(spec.outer_size), + _reduce_size(spec.reduce_size), + _inner_size(spec.inner_size), + _aggr(spec.aggr) { } @@ -133,24 +238,25 @@ DenseSingleReduceFunction::~DenseSingleReduceFunction() = default; InterpretedFunction::Instruction DenseSingleReduceFunction::compile_self(eval::EngineOrFactory, Stash &stash) const { - auto ¶ms = stash.create<Params>(result_type(), child().result_type(), _dim_idx); auto op = typify_invoke<4,MyTypify,MyGetFun>(result_type().cell_type(), _aggr, - (params.dim_size >= 8), (params.inner_size == 1)); + (_reduce_size >= 8), (_inner_size == 1)); + auto ¶ms = stash.create<Params>(result_type(), _outer_size, _reduce_size, _inner_size); return InterpretedFunction::Instruction(op, wrap_param<Params>(params)); } const TensorFunction & DenseSingleReduceFunction::optimize(const TensorFunction &expr, Stash &stash) { - auto reduce = as<Reduce>(expr); - if (reduce && (reduce->dimensions().size() == 1) && - reduce->child().result_type().is_dense() && - expr.result_type().is_dense()) - { - size_t dim_idx = reduce->child().result_type().dimension_index(reduce->dimensions()[0]); - assert(dim_idx != ValueType::Dimension::npos); - assert(expr.result_type().cell_type() == reduce->child().result_type().cell_type()); - return stash.create<DenseSingleReduceFunction>(expr.result_type(), reduce->child(), dim_idx, reduce->aggr()); + if (auto reduce = as<Reduce>(expr)) { + const auto &child = reduce->child(); + auto spec_list = make_dense_single_reduce_list(child.result_type(), reduce->aggr(), reduce->dimensions()); + if (!spec_list.empty()) { + const auto *prev = &child; + for (const auto &spec: spec_list) { + prev = &stash.create<DenseSingleReduceFunction>(spec, *prev); + } + return *prev; + } } return expr; } diff --git a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.h b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.h index 7f9313df600..f2db3155290 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.h +++ b/eval/src/vespa/eval/tensor/dense/dense_single_reduce_function.h @@ -6,22 +6,46 @@ namespace vespalib::tensor { +struct DenseSingleReduceSpec { + eval::ValueType result_type; + size_t outer_size; + size_t reduce_size; + size_t inner_size; + eval::Aggr aggr; +}; + +/** + * Decompose the specified reduce operation into a sequence of single + * dense reduce operations. Returns an empty list if decomposition + * fails. + **/ +std::vector<DenseSingleReduceSpec> +make_dense_single_reduce_list(const eval::ValueType &type, eval::Aggr aggr, + const std::vector<vespalib::string> &reduce_dims); + /** - * Tensor function reducing a single dimension of a dense - * tensor where the result is also a dense tensor. + * Tensor function reducing a single dimension of a dense tensor where + * the result is also a dense tensor. The optimize function may create + * multiple tensor functions to compose a multi-stage reduce + * operation. Adjacent reduced dimensions will be handled is if they + * were a single dimension. Trivial dimensions will be trivially + * reduced along with any other dimension. **/ class DenseSingleReduceFunction : public eval::tensor_function::Op1 { private: - size_t _dim_idx; + size_t _outer_size; + size_t _reduce_size; + size_t _inner_size; eval::Aggr _aggr; public: - DenseSingleReduceFunction(const eval::ValueType &result_type, - const eval::TensorFunction &child, - size_t dim_idx, eval::Aggr aggr); + DenseSingleReduceFunction(const DenseSingleReduceSpec &spec, + const eval::TensorFunction &child); ~DenseSingleReduceFunction() override; - size_t dim_idx() const { return _dim_idx; } + size_t outer_size() const { return _outer_size; } + size_t reduce_size() const { return _reduce_size; } + size_t inner_size() const { return _inner_size; } eval::Aggr aggr() const { return _aggr; } bool result_is_mutable() const override { return true; } eval::InterpretedFunction::Instruction compile_self(eval::EngineOrFactory engine, Stash &stash) const override; diff --git a/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp b/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp index 5db533a4655..c49809f265f 100644 --- a/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp +++ b/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp @@ -19,6 +19,7 @@ LOG_SETUP(".eval.onnx_wrapper"); using vespalib::ArrayRef; using vespalib::ConstArrayRef; +using vespalib::eval::CellType; using vespalib::eval::ValueType; using vespalib::eval::TypifyCellType; @@ -110,18 +111,18 @@ auto convert_optimize(Onnx::Optimize optimize) { abort(); } -ValueType::CellType to_cell_type(Onnx::ElementType type) { +CellType to_cell_type(Onnx::ElementType type) { switch (type) { case Onnx::ElementType::INT8: [[fallthrough]]; case Onnx::ElementType::INT16: [[fallthrough]]; case Onnx::ElementType::UINT8: [[fallthrough]]; case Onnx::ElementType::UINT16: [[fallthrough]]; - case Onnx::ElementType::FLOAT: return ValueType::CellType::FLOAT; + case Onnx::ElementType::FLOAT: return CellType::FLOAT; case Onnx::ElementType::INT32: [[fallthrough]]; case Onnx::ElementType::INT64: [[fallthrough]]; case Onnx::ElementType::UINT32: [[fallthrough]]; case Onnx::ElementType::UINT64: [[fallthrough]]; - case Onnx::ElementType::DOUBLE: return ValueType::CellType::DOUBLE; + case Onnx::ElementType::DOUBLE: return CellType::DOUBLE; } abort(); } @@ -381,21 +382,21 @@ Onnx::EvalContext::convert_result(EvalContext &self, size_t idx) struct Onnx::EvalContext::SelectAdaptParam { template <typename ...Ts> static auto invoke() { return adapt_param<Ts...>; } - auto operator()(eval::ValueType::CellType ct) { + auto operator()(eval::CellType ct) { return typify_invoke<1,MyTypify,SelectAdaptParam>(ct); } }; struct Onnx::EvalContext::SelectConvertParam { template <typename ...Ts> static auto invoke() { return convert_param<Ts...>; } - auto operator()(eval::ValueType::CellType ct, Onnx::ElementType et) { + auto operator()(eval::CellType ct, Onnx::ElementType et) { return typify_invoke<2,MyTypify,SelectConvertParam>(ct, et); } }; struct Onnx::EvalContext::SelectConvertResult { template <typename ...Ts> static auto invoke() { return convert_result<Ts...>; } - auto operator()(Onnx::ElementType et, eval::ValueType::CellType ct) { + auto operator()(Onnx::ElementType et, eval::CellType ct) { return typify_invoke<2,MyTypify,SelectConvertResult>(et, ct); } }; diff --git a/eval/src/vespa/eval/tensor/dense/typed_cells_dispatch.h b/eval/src/vespa/eval/tensor/dense/typed_cells_dispatch.h index 87b1a5b47ed..84ce8749eb4 100644 --- a/eval/src/vespa/eval/tensor/dense/typed_cells_dispatch.h +++ b/eval/src/vespa/eval/tensor/dense/typed_cells_dispatch.h @@ -6,7 +6,7 @@ namespace vespalib::tensor { -using CellType = vespalib::eval::ValueType::CellType; +using vespalib::eval::CellType; using TypedCells = vespalib::eval::TypedCells; template <typename TGT, typename... Args> diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp index 13c4711668b..837b135c0aa 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp @@ -8,7 +8,7 @@ using vespalib::nbostream; using vespalib::eval::ValueType; -using CellType = vespalib::eval::ValueType::CellType; +using vespalib::eval::CellType; namespace vespalib::tensor { diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h index 21618dcb6ce..f0516e9fcc9 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h @@ -18,7 +18,7 @@ class DenseTensorView; class DenseBinaryFormat { public: - using CellType = eval::ValueType::CellType; + using CellType = vespalib::eval::CellType; static void serialize(nbostream &stream, const DenseTensorView &tensor); static std::unique_ptr<DenseTensorView> deserialize(nbostream &stream, CellType cell_type); diff --git a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp index eda8f7eecc7..a4022c4f60a 100644 --- a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.cpp @@ -12,8 +12,8 @@ #include <cassert> using vespalib::nbostream; +using vespalib::eval::CellType; using vespalib::eval::ValueType; -using CellType = vespalib::eval::ValueType::CellType; namespace vespalib::tensor { diff --git a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h index 0611d7d5a23..d4c7fa4bf6f 100644 --- a/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/sparse_binary_format.h @@ -17,7 +17,7 @@ class Tensor; class SparseBinaryFormat { public: - using CellType = eval::ValueType::CellType; + using CellType = eval::CellType; static void serialize(nbostream &stream, const Tensor &tensor); static std::unique_ptr<Tensor> deserialize(nbostream &stream, CellType cell_type); diff --git a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp index 758ceb43ab4..2d3d1f4a0ea 100644 --- a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp @@ -19,7 +19,7 @@ LOG_SETUP(".eval.tensor.serialization.typed_binary_format"); using vespalib::nbostream; using vespalib::eval::ValueType; -using CellType = vespalib::eval::ValueType::CellType; +using vespalib::eval::CellType; namespace vespalib::tensor { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index cd60a082472..8835129ba93 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -343,6 +343,13 @@ public class Flags { "Takes effect on next internal redeployment", APPLICATION_ID); + public static final UnboundBooleanFlag USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING = defineFeatureFlag( + "use-power-of-two-choices-load-balancing", + false, + "Whether to use Power of two load balancing algorithm for application", + "Takes effect on next internal redeployment", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index d074915a023..3a848b33c76 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -281,12 +281,12 @@ public abstract class ControllerHttpClient { if (response.statusCode() / 100 == 4) throw new IllegalArgumentException("Bad request for " + request + ": " + message); - throw new IOException("Failed " + request + ": " + message); + throw new IOException(message); } catch (IOException e) { // Catches the above, and timeout exceptions from the client. if (thrown == null) - thrown = new UncheckedIOException(e); + thrown = new UncheckedIOException("Failed " + request + ": " + e, e); else thrown.addSuppressed(e); diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilter.java new file mode 100644 index 00000000000..b891212031f --- /dev/null +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilter.java @@ -0,0 +1,21 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.jdisc.http.filter.security.misc; + +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; + +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.Optional; + +public class VespaTlsFilter extends JsonSecurityRequestFilterBase { + + @Override + protected Optional<ErrorResponse> filter(DiscFilterRequest request) { + return request.getClientCertificateChain().isEmpty() + ? Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Forbidden to access this path")) + : Optional.empty(); + } +} diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilterTest.java new file mode 100644 index 00000000000..294126eb349 --- /dev/null +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/misc/VespaTlsFilterTest.java @@ -0,0 +1,66 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.jdisc.http.filter.security.misc; + +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.net.URI; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.when; + +public class VespaTlsFilterTest { + + @Test + public void testFilter() { + assertSuccess(createRequest(List.of(createCertificate()))); + assertForbidden(createRequest(Collections.emptyList())); + } + + private static X509Certificate createCertificate() { + return X509CertificateBuilder + .fromKeypair( + KeyUtils.generateKeypair(KeyAlgorithm.EC), new X500Principal("CN=test"), + Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), + SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(1)) + .build(); + } + + private static DiscFilterRequest createRequest(List<X509Certificate> certChain) { + DiscFilterRequest request = Mockito.mock(DiscFilterRequest.class); + when(request.getClientCertificateChain()).thenReturn(certChain); + when(request.getMethod()).thenReturn("GET"); + when(request.getUri()).thenReturn(URI.create("http://localhost:8080/")); + return request; + } + + private static void assertForbidden(DiscFilterRequest request) { + VespaTlsFilter filter = new VespaTlsFilter(); + RequestHandlerTestDriver.MockResponseHandler handler = new RequestHandlerTestDriver.MockResponseHandler(); + filter.filter(request, handler); + assertEquals(Response.Status.FORBIDDEN, handler.getStatus()); + } + + private static void assertSuccess(DiscFilterRequest request) { + VespaTlsFilter filter = new VespaTlsFilter(); + RequestHandlerTestDriver.MockResponseHandler handler = new RequestHandlerTestDriver.MockResponseHandler(); + filter.filter(request, handler); + assertNull(handler.getResponse()); + } +} diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index 8bf7f30964a..8b48631d4aa 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -742,6 +742,7 @@ "public com.yahoo.jdisc.http.ServerConfig$Builder filter(java.util.List)", "public com.yahoo.jdisc.http.ServerConfig$Builder defaultFilters(com.yahoo.jdisc.http.ServerConfig$DefaultFilters$Builder)", "public com.yahoo.jdisc.http.ServerConfig$Builder defaultFilters(java.util.List)", + "public com.yahoo.jdisc.http.ServerConfig$Builder strictFiltering(boolean)", "public com.yahoo.jdisc.http.ServerConfig$Builder maxWorkerThreads(int)", "public com.yahoo.jdisc.http.ServerConfig$Builder minWorkerThreads(int)", "public com.yahoo.jdisc.http.ServerConfig$Builder stopTimeout(double)", @@ -930,6 +931,7 @@ "public com.yahoo.jdisc.http.ServerConfig$Filter filter(int)", "public java.util.List defaultFilters()", "public com.yahoo.jdisc.http.ServerConfig$DefaultFilters defaultFilters(int)", + "public boolean strictFiltering()", "public int maxWorkerThreads()", "public int minWorkerThreads()", "public double stopTimeout()", diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/FilterResolver.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/FilterResolver.java index b80dd216b04..1e2686aa184 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/FilterResolver.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/FilterResolver.java @@ -2,6 +2,12 @@ package com.yahoo.jdisc.http.server.jetty; import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.NoopSharedResource; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.FastContentWriter; +import com.yahoo.jdisc.handler.ResponseDispatch; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.RequestFilter; import com.yahoo.jdisc.http.filter.ResponseFilter; import com.yahoo.jdisc.http.servlet.ServletRequest; @@ -22,10 +28,12 @@ class FilterResolver { private final FilterBindings bindings; private final Metric metric; + private final boolean strictFiltering; - FilterResolver(FilterBindings bindings, Metric metric) { + FilterResolver(FilterBindings bindings, Metric metric, boolean strictFiltering) { this.bindings = bindings; this.metric = metric; + this.strictFiltering = strictFiltering; } Optional<RequestFilter> resolveRequestFilter(HttpServletRequest servletRequest, URI jdiscUri) { @@ -33,8 +41,13 @@ class FilterResolver { if (maybeFilterId.isPresent()) { metric.add(MetricDefinitions.FILTERING_REQUEST_HANDLED, 1L, createMetricContext(servletRequest, maybeFilterId.get())); servletRequest.setAttribute(ServletRequest.JDISC_REQUEST_CHAIN, maybeFilterId.get()); - } else { + } else if (!strictFiltering) { metric.add(MetricDefinitions.FILTERING_REQUEST_UNHANDLED, 1L, createMetricContext(servletRequest, null)); + } else { + String syntheticFilterId = RejectingRequestFilter.SYNTHETIC_FILTER_CHAIN_ID; + metric.add(MetricDefinitions.FILTERING_REQUEST_HANDLED, 1L, createMetricContext(servletRequest, syntheticFilterId)); + servletRequest.setAttribute(ServletRequest.JDISC_REQUEST_CHAIN, syntheticFilterId); + return Optional.of(RejectingRequestFilter.INSTANCE); } return maybeFilterId.map(bindings::getRequestFilter); } @@ -56,4 +69,20 @@ class FilterResolver { : Map.of(); return JDiscHttpServlet.getConnector(request).createRequestMetricContext(request, extraDimensions); } + + private static class RejectingRequestFilter extends NoopSharedResource implements RequestFilter { + + private static final RejectingRequestFilter INSTANCE = new RejectingRequestFilter(); + private static final String SYNTHETIC_FILTER_CHAIN_ID = "strict-reject"; + + @Override + public void filter(HttpRequest request, ResponseHandler handler) { + Response response = new Response(Response.Status.FORBIDDEN); + response.headers().add("Content-Type", "text/plain"); + try (FastContentWriter writer = ResponseDispatch.newInstance(response).connectFastWriter(handler)) { + writer.write("Request did not match any request filter chain"); + } + } + } + } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscContext.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscContext.java index 66471587bd5..b37a7352dc6 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscContext.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscContext.java @@ -20,7 +20,7 @@ public class JDiscContext { Metric metric, ServerConfig serverConfig) { - this.filterResolver = new FilterResolver(filterBindings, metric); + this.filterResolver = new FilterResolver(filterBindings, metric, serverConfig.strictFiltering()); this.container = container; this.janitor = janitor; this.metric = metric; diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def index f33dc35ea0b..f75a4aaa441 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def @@ -33,6 +33,9 @@ defaultFilters[].filterId string # The local port which the default filter should be applied to defaultFilters[].localPort int +# Reject all requests not handled by a request filter (chain) +strictFiltering bool default = false + # Max number of threads in underlying Jetty pool maxWorkerThreads int default = 200 diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/FilterTestCase.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/FilterTestCase.java index fd929b3e037..9c5c4027ae3 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/FilterTestCase.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/FilterTestCase.java @@ -35,6 +35,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -495,7 +496,7 @@ public class FilterTestCase { .build(); MetricConsumerMock metricConsumerMock = new MetricConsumerMock(); MyRequestHandler requestHandler = new MyRequestHandler(); - TestDriver testDriver = newDriver(requestHandler, filterBindings, metricConsumerMock); + TestDriver testDriver = newDriver(requestHandler, filterBindings, metricConsumerMock, false); testDriver.client().get("/status.html"); assertThat(requestHandler.awaitInvocation(), is(true)); @@ -510,25 +511,47 @@ public class FilterTestCase { assertThat(testDriver.close(), is(true)); } + @Test + public void requireThatStrictFilteringRejectsRequestsNotMatchingFilterChains() throws IOException { + RequestFilter filter = mock(RequestFilter.class); + FilterBindings filterBindings = new FilterBindings.Builder() + .addRequestFilter("my-request-filter", filter) + .addRequestFilterBinding("my-request-filter", "http://*/filtered/*") + .build(); + MyRequestHandler requestHandler = new MyRequestHandler(); + TestDriver testDriver = newDriver(requestHandler, filterBindings, new MetricConsumerMock(), true); + + testDriver.client().get("/unfiltered/") + .expectStatusCode(is(Response.Status.FORBIDDEN)) + .expectContent(containsString("Request did not match any request filter chain")); + verify(filter, never()).filter(any(), any()); + assertThat(testDriver.close(), is(true)); + } + private static TestDriver newDriver(MyRequestHandler requestHandler, FilterBindings filterBindings) { - return newDriver(requestHandler, filterBindings, new MetricConsumerMock()); + return newDriver(requestHandler, filterBindings, new MetricConsumerMock(), false); } - private static TestDriver newDriver(MyRequestHandler requestHandler, FilterBindings filterBindings, MetricConsumerMock metricConsumer) { + private static TestDriver newDriver( + MyRequestHandler requestHandler, + FilterBindings filterBindings, + MetricConsumerMock metricConsumer, + boolean strictFiltering) { return TestDriver.newInstance( JettyHttpServer.class, requestHandler, - newFilterModule(filterBindings, metricConsumer)); + newFilterModule(filterBindings, metricConsumer, strictFiltering)); } - private static com.google.inject.Module newFilterModule(FilterBindings filterBindings, MetricConsumerMock metricConsumer) { + private static com.google.inject.Module newFilterModule( + FilterBindings filterBindings, MetricConsumerMock metricConsumer, boolean strictFiltering) { return Modules.combine( new AbstractModule() { @Override protected void configure() { bind(FilterBindings.class).toInstance(filterBindings); - bind(ServerConfig.class).toInstance(new ServerConfig(new ServerConfig.Builder())); + bind(ServerConfig.class).toInstance(new ServerConfig(new ServerConfig.Builder().strictFiltering(strictFiltering))); bind(ConnectorConfig.class).toInstance(new ConnectorConfig(new ConnectorConfig.Builder())); bind(ServletPathsConfig.class).toInstance(new ServletPathsConfig(new ServletPathsConfig.Builder())); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index d0ee6229428..00327dc0002 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -94,7 +94,7 @@ public final class Node implements Nodelike { requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); if (parentHostname.isPresent()) { - if (!ipConfig.pool().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); + if (!ipConfig.pool().getIpSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); if (modelName.isPresent()) throw new IllegalArgumentException("A child node cannot have model name set"); if (switchHostname.isPresent()) throw new IllegalArgumentException("A child node cannot have switch hostname set"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 663d1d19995..03ff89d36dc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -460,7 +460,7 @@ public class NodeRepository extends AbstractComponent { .map(node -> { if (node.state() != State.provisioned && node.state() != State.dirty) illegal("Can not set " + node + " ready. It is not provisioned or dirty."); - if (node.type() == NodeType.host && node.ipConfig().pool().isEmpty()) + if (node.type() == NodeType.host && node.ipConfig().pool().getIpSet().isEmpty()) illegal("Can not set host " + node + " ready. Its IP address pool is empty."); return node.withWantToRetire(false, false, Agent.system, clock.instant()); }) @@ -503,12 +503,6 @@ public class NodeRepository extends AbstractComponent { } } - /** Deactivate nodes owned by application guarded by given lock */ - public void deactivate(ApplicationTransaction transaction) { - deactivate(db.readNodes(transaction.application(), State.reserved, State.active), transaction); - applications.remove(transaction); - } - /** * Deactivates these nodes in a transaction and returns the nodes in the new state which will hold if the * transaction commits. @@ -517,6 +511,19 @@ public class NodeRepository extends AbstractComponent { return db.writeTo(State.inactive, nodes, Agent.application, Optional.empty(), transaction.nested()); } + /** Removes this application: Active nodes are deactivated while all non-active nodes are set dirty. */ + public void remove(ApplicationTransaction transaction) { + NodeList applicationNodes = list(transaction.application()); + NodeList activeNodes = applicationNodes.state(State.active); + deactivate(activeNodes.asList(), transaction); + db.writeTo(State.dirty, + applicationNodes.except(activeNodes.asSet()).asList(), + Agent.system, + Optional.of("Application is removed"), + transaction.nested()); + applications.remove(transaction); + } + /** Move nodes to the dirty state */ public List<Node> setDirty(List<Node> nodes, Agent agent, String reason) { return performOn(NodeListFilter.from(nodes), (node, lock) -> setDirty(node, agent, reason)); @@ -532,6 +539,7 @@ public class NodeRepository extends AbstractComponent { return db.writeTo(State.dirty, node, agent, Optional.of(reason)); } + public List<Node> dirtyRecursively(String hostname, Agent agent, String reason) { Node nodeToDirty = getNode(hostname).orElseThrow(() -> new IllegalArgumentException("Could not deallocate " + hostname + ": Node not found")); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java index fd92b5b0ca0..847b825a7a4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -57,7 +57,7 @@ public class Application { public Application withCluster(ClusterSpec.Id id, boolean exclusive, ClusterResources min, ClusterResources max) { Cluster cluster = clusters.get(id); if (cluster == null) - cluster = new Cluster(id, exclusive, min, max, Optional.empty(), Optional.empty(), List.of()); + cluster = new Cluster(id, exclusive, min, max, Optional.empty(), Optional.empty(), List.of(), ""); else cluster = cluster.withConfiguration(exclusive, min, max); return with(cluster); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index a17ee081447..90133f7499e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -25,6 +25,7 @@ public class Cluster { private final Optional<ClusterResources> suggested; private final Optional<ClusterResources> target; private final List<ScalingEvent> scalingEvents; + private final String autoscalingStatus; public Cluster(ClusterSpec.Id id, boolean exclusive, @@ -32,7 +33,8 @@ public class Cluster { ClusterResources maxResources, Optional<ClusterResources> suggestedResources, Optional<ClusterResources> targetResources, - List<ScalingEvent> scalingEvents) { + List<ScalingEvent> scalingEvents, + String autoscalingStatus) { this.id = Objects.requireNonNull(id); this.exclusive = exclusive; this.min = Objects.requireNonNull(minResources); @@ -44,6 +46,7 @@ public class Cluster { else this.target = targetResources; this.scalingEvents = scalingEvents; + this.autoscalingStatus = autoscalingStatus; } public ClusterSpec.Id id() { return id; } @@ -73,21 +76,33 @@ public class Cluster { /** Returns the recent scaling events in this cluster */ public List<ScalingEvent> scalingEvents() { return scalingEvents; } + public Optional<ScalingEvent> lastScalingEvent() { + if (scalingEvents.isEmpty()) return Optional.empty(); + return Optional.of(scalingEvents.get(scalingEvents.size() - 1)); + } + + /** The latest autoscaling status of this cluster, or empty (never null) if none */ + public String autoscalingStatus() { return autoscalingStatus; } + public Cluster withConfiguration(boolean exclusive, ClusterResources min, ClusterResources max) { - return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents); + return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus); } public Cluster withSuggested(Optional<ClusterResources> suggested) { - return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents); + return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus); } public Cluster withTarget(Optional<ClusterResources> target) { - return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents); + return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus); } public Cluster with(ScalingEvent scalingEvent) { // NOTE: We're just storing the latest scaling event so far - return new Cluster(id, exclusive, min, max, suggested, target, List.of(scalingEvent)); + return new Cluster(id, exclusive, min, max, suggested, target, List.of(scalingEvent), autoscalingStatus); + } + + public Cluster withAutoscalingStatus(String autoscalingStatus) { + return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index b7729577bda..c4f11ee76d0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -3,14 +3,16 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; import java.time.Duration; +import java.time.Instant; import java.util.List; +import java.util.Objects; import java.util.Optional; -import java.util.logging.Logger; /** * The autoscaler makes decisions about the flavor and node count that should be allocated to a cluster @@ -20,8 +22,6 @@ import java.util.logging.Logger; */ public class Autoscaler { - private final Logger log = Logger.getLogger(this.getClass().getName()); - /** What cost difference factor is worth a reallocation? */ private static final double costDifferenceWorthReallocation = 0.1; /** What difference factor for a resource is worth a reallocation? */ @@ -55,40 +55,37 @@ public class Autoscaler { * @return scaling advice for this cluster */ public Advice autoscale(Cluster cluster, List<Node> clusterNodes) { - if (cluster.minResources().equals(cluster.maxResources())) return Advice.none(); // Shortcut + if (cluster.minResources().equals(cluster.maxResources())) return Advice.none("Autoscaling is disabled"); // Shortcut return autoscale(cluster, clusterNodes, Limits.of(cluster), cluster.exclusive()); } private Advice autoscale(Cluster cluster, List<Node> clusterNodes, Limits limits, boolean exclusive) { - log.fine(() -> "Autoscale " + cluster.toString()); - - if (unstable(clusterNodes, nodeRepository)) { - log.fine(() -> "Unstable - Advice.none " + cluster.toString()); - return Advice.none(); - } + if (unstable(clusterNodes, nodeRepository)) + return Advice.none("Cluster change in progress"); - AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, nodeRepository, cluster.exclusive()); + AllocatableClusterResources currentAllocation = + new AllocatableClusterResources(clusterNodes, nodeRepository, cluster.exclusive()); ClusterTimeseries clusterTimeseries = new ClusterTimeseries(cluster, clusterNodes, metricsDb, nodeRepository); Optional<Double> cpuLoad = clusterTimeseries.averageLoad(Resource.cpu, cluster); Optional<Double> memoryLoad = clusterTimeseries.averageLoad(Resource.memory, cluster); Optional<Double> diskLoad = clusterTimeseries.averageLoad(Resource.disk, cluster); - if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) { - return Advice.none(); - } + if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) + return Advice.none("Collecting more data before making new scaling decisions"); + var target = ResourceTarget.idealLoad(cpuLoad.get(), memoryLoad.get(), diskLoad.get(), currentAllocation); Optional<AllocatableClusterResources> bestAllocation = allocationOptimizer.findBestAllocation(target, currentAllocation, limits, exclusive); - if (bestAllocation.isEmpty()) { - log.fine(() -> "bestAllocation.isEmpty: Advice.dontScale for " + cluster.toString()); - return Advice.dontScale(); - } - if (similar(bestAllocation.get(), currentAllocation)) { - log.fine(() -> "Current allocation similar: Advice.dontScale for " + cluster.toString()); - return Advice.dontScale(); - } + if (bestAllocation.isEmpty()) + return Advice.dontScale("No allocation changes are possible within configured limits"); + + if (similar(bestAllocation.get(), currentAllocation)) + return Advice.dontScale("Cluster is ideally scaled (within configured limits)"); + if (isDownscaling(bestAllocation.get(), currentAllocation) && recentlyScaled(cluster, clusterNodes)) + return Advice.dontScale("Waiting a while before scaling down"); + return Advice.scaleTo(bestAllocation.get().toAdvertisedClusterResources()); } @@ -107,10 +104,23 @@ public class Autoscaler { return Math.abs(r1 - r2) / (( r1 + r2) / 2) < threshold; } + /** Returns true if this reduces total resources in any dimension */ + private boolean isDownscaling(AllocatableClusterResources target, AllocatableClusterResources current) { + NodeResources targetTotal = target.toAdvertisedClusterResources().totalResources(); + NodeResources currentTotal = current.toAdvertisedClusterResources().totalResources(); + return ! targetTotal.justNumbers().satisfies(currentTotal.justNumbers()); + } + + private boolean recentlyScaled(Cluster cluster, List<Node> clusterNodes) { + Duration downscalingDelay = downscalingDelay(clusterNodes.get(0).allocation().get().membership().cluster().type()); + return cluster.lastScalingEvent().map(event -> event.at()).orElse(Instant.MIN) + .isAfter(nodeRepository.clock().instant().minus(downscalingDelay)); + } + /** The duration of the window we need to consider to make a scaling decision. See also minimumMeasurementsPerNode */ static Duration scalingWindow(ClusterSpec.Type clusterType) { if (clusterType.isContent()) return Duration.ofHours(12); - return Duration.ofHours(1); + return Duration.ofMinutes(30); } static Duration maxScalingWindow() { @@ -120,7 +130,16 @@ public class Autoscaler { /** Measurements are currently taken once a minute. See also scalingWindow */ static int minimumMeasurementsPerNode(ClusterSpec.Type clusterType) { if (clusterType.isContent()) return 60; - return 20; + return 7; + } + + /** + * We should wait a while before scaling down after a scaling event as a peak in usage + * indicates more peaks may arrive in the near future. + */ + static Duration downscalingDelay(ClusterSpec.Type clusterType) { + if (clusterType.isContent()) return Duration.ofHours(12); + return Duration.ofHours(1); } public static boolean unstable(List<Node> nodes, NodeRepository nodeRepository) { @@ -141,10 +160,12 @@ public class Autoscaler { private final boolean present; private final Optional<ClusterResources> target; + private final String reason; - private Advice(Optional<ClusterResources> target, boolean present) { + private Advice(Optional<ClusterResources> target, boolean present, String reason) { this.target = target; this.present = present; + this.reason = Objects.requireNonNull(reason); } /** @@ -159,10 +180,14 @@ public class Autoscaler { /** True if this provides advice (which may be to keep the current allocation) */ public boolean isPresent() { return present; } - private static Advice none() { return new Advice(Optional.empty(), false); } - private static Advice dontScale() { return new Advice(Optional.empty(), true); } - private static Advice scaleTo(ClusterResources target) { return new Advice(Optional.of(target), true); } + /** The reason for this advice */ + public String reason() { return reason; } + private static Advice none(String reason) { return new Advice(Optional.empty(), false, reason); } + private static Advice dontScale(String reason) { return new Advice(Optional.empty(), true, reason); } + private static Advice scaleTo(ClusterResources target) { + return new Advice(Optional.of(target), true, "Scaling due to load changes"); + } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java index bb91b77dce5..3c93e7ee7f6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterTimeseries.java @@ -77,16 +77,8 @@ public class ClusterTimeseries { // Require a total number of measurements scaling with the number of nodes, // but don't require that we have at least that many from every node int measurementCount = currentMeasurements.stream().mapToInt(m -> m.size()).sum(); - if (measurementCount / clusterNodes.size() < Autoscaler.minimumMeasurementsPerNode(clusterType)) { - log.fine(() -> "Too few measurements per node for " + cluster.toString() + ": measurementCount " + measurementCount + - " (" + nodeTimeseries.stream().mapToInt(m -> m.size()).sum() + " before filtering"); - return Optional.empty(); - } - if (currentMeasurements.size() != clusterNodes.size()) { - log.fine(() -> "Mssing measurements from some nodes for " + cluster.toString() + ": Has from " + currentMeasurements.size() + - "but need " + clusterNodes.size() + "(before filtering: " + nodeTimeseries.size() + ")"); - return Optional.empty(); - } + if (measurementCount / clusterNodes.size() < Autoscaler.minimumMeasurementsPerNode(clusterType)) return Optional.empty(); + if (currentMeasurements.size() != clusterNodes.size()) return Optional.empty(); double measurementSum = currentMeasurements.stream().flatMap(m -> m.asList().stream()).mapToDouble(m -> value(resource, m)).sum(); return Optional.of(measurementSum / measurementCount); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 0b8cb4f635b..809c54146d0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -17,7 +17,6 @@ import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import java.time.Duration; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -57,7 +56,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { private void autoscale(ApplicationId application, List<Node> applicationNodes) { try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) { - if ( ! deployment.isValid()) return; // Another config server will consider this application + if ( ! deployment.isValid()) return; nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId, clusterNodes, deployment)); } } @@ -70,13 +69,13 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { Optional<Cluster> cluster = application.cluster(clusterId); if (cluster.isEmpty()) return; - log.fine(() -> "Autoscale " + application.toString()); - var advice = autoscaler.autoscale(cluster.get(), clusterNodes); - if (advice.isEmpty()) return; - - if ( ! cluster.get().targetResources().equals(advice.target())) { + application = application.with(cluster.get().withAutoscalingStatus(advice.reason())); + if (advice.isEmpty()) { + applications().put(application, deployment.applicationLock().get()); + } + else if ( ! cluster.get().targetResources().equals(advice.target())) { applications().put(application.with(cluster.get().withTarget(advice.target())), deployment.applicationLock().get()); if (advice.target().isPresent()) { logAutoscaling(advice.target().get(), applicationId, cluster.get(), clusterNodes); @@ -100,11 +99,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { } static String toString(ClusterResources r) { - return String.format(Locale.US, "%d%s * [vcpu: %.1f, memory: %.1f Gb, disk %.1f Gb]" + - " (total: [vcpu: %.1f, memory: %.1f Gb, disk: %.1f Gb])", - r.nodes(), r.groups() > 1 ? " (in " + r.groups() + " groups)" : "", - r.nodeResources().vcpu(), r.nodeResources().memoryGb(), r.nodeResources().diskGb(), - r.nodes() * r.nodeResources().vcpu(), r.nodes() * r.nodeResources().memoryGb(), r.nodes() * r.nodeResources().diskGb()); + return r + " (total: " + r.totalResources() + ")"; } private Map<ClusterSpec.Id, List<Node>> nodesByCluster(List<Node> applicationNodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 3bf287a3e80..064569a827a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -70,6 +71,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { updateLockMetrics(); updateDockerMetrics(nodes); updateTenantUsageMetrics(nodes); + updateRepairTicketMetrics(nodes); return true; } @@ -297,6 +299,15 @@ public class MetricsReporter extends NodeRepositoryMaintainer { ); } + private void updateRepairTicketMetrics(NodeList nodes) { + nodes.nodeType(NodeType.host).stream() + .map(node -> node.reports().getReport("repairTicket")) + .flatMap(Optional::stream) + .map(report -> report.getInspector().field("status").asString()) + .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())) + .forEach((status, number) -> metric.set("hostedVespa.breakfixedHosts", number, getContextAt("status", status))); + } + private static NodeResources getCapacityTotal(NodeList nodes) { return nodes.hosts().state(active).asList().stream() .map(host -> host.flavor().resources()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 41d6c1e5425..bac31c40418 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -20,6 +20,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.config.provision.NodeType.confighost; import static com.yahoo.config.provision.NodeType.controllerhost; @@ -254,18 +255,25 @@ public class IP { * @return an allocation from the pool, if any can be made */ public Optional<Allocation> findAllocation(LockedNodeList nodes, NameResolver resolver) { + if (ipAddresses.asSet().isEmpty()) { + // IP addresses have not yet been resolved and should be done later. + return findUnusedAddressStream(nodes) + .map(Allocation::ofAddress) + .findFirst(); + } + if (ipAddresses.protocol == IpAddresses.Protocol.ipv4) { - return findUnused(nodes).stream() + return findUnusedIpAddresses(nodes).stream() .findFirst() .map(addr -> Allocation.ofIpv4(addr, resolver)); } - var unusedAddresses = findUnused(nodes); + var unusedAddresses = findUnusedIpAddresses(nodes); var allocation = unusedAddresses.stream() .filter(IP::isV6) .findFirst() .map(addr -> Allocation.ofIpv6(addr, resolver)); - allocation.flatMap(Allocation::secondary).ifPresent(ipv4Address -> { + allocation.flatMap(Allocation::ipv4Address).ifPresent(ipv4Address -> { if (!unusedAddresses.contains(ipv4Address)) { throw new IllegalArgumentException("Allocation resolved " + ipv4Address + " from hostname " + allocation.get().hostname + @@ -276,17 +284,43 @@ public class IP { } /** - * Finds all unused addresses in this pool + * Finds all unused IP addresses in this pool * * @param nodes a list of all nodes in the repository */ - public Set<String> findUnused(NodeList nodes) { + public Set<String> findUnusedIpAddresses(NodeList nodes) { var unusedAddresses = new LinkedHashSet<>(getIpSet()); nodes.matching(node -> node.ipConfig().primary().stream().anyMatch(ip -> getIpSet().contains(ip))) .forEach(node -> unusedAddresses.removeAll(node.ipConfig().primary())); return Collections.unmodifiableSet(unusedAddresses); } + /** + * Returns the number of unused IP addresses in the pool, assuming any and all unaccounted for hostnames + * in the pool are resolved to exactly 1 IP address (or 2 with {@link IpAddresses.Protocol#dualStack}). + */ + public int eventuallyUnusedAddressCount(NodeList nodes) { + // The address pool is filled immediately upon provisioning in dynamically provisioned zones, + // and within short time the IP address pool is filled. For all other cases, the IP address + // pool is already filled. + // + // The count in this method relies on the size of the IP address pool if that's non-empty, + // otherwise fall back to the address/hostname pool. + + + Set<String> currentIpAddresses = this.ipAddresses.asSet(); + if (!currentIpAddresses.isEmpty()) { + return findUnusedIpAddresses(nodes).size(); + } + + return (int) findUnusedAddressStream(nodes).count(); + } + + private Stream<Address> findUnusedAddressStream(NodeList nodes) { + Set<String> hostnames = nodes.stream().map(Node::hostname).collect(Collectors.toSet()); + return addresses.stream().filter(address -> !hostnames.contains(address.hostname())); + } + public IpAddresses.Protocol getProtocol() { return ipAddresses.protocol; } @@ -299,10 +333,6 @@ public class IP { return addresses; } - public boolean isEmpty() { - return getIpSet().isEmpty(); - } - public Pool withIpAddresses(Set<String> ipAddresses) { return Pool.of(ipAddresses, addresses); } @@ -326,22 +356,17 @@ public class IP { } - /** An IP address allocation from a pool */ + /** An address allocation from a pool */ public static class Allocation { private final String hostname; - private final String primary; - private final Optional<String> secondary; - - private Allocation(String hostname, String primary, Optional<String> secondary) { - Objects.requireNonNull(primary, "primary must be non-null"); - Objects.requireNonNull(secondary, "ipv4Address must be non-null"); - if (secondary.isPresent() && !isV4(secondary.get())) { // Secondary must be IPv4, if present - throw new IllegalArgumentException("Invalid IPv4 address '" + secondary + "'"); - } + private final Optional<String> ipv4Address; + private final Optional<String> ipv6Address; + + private Allocation(String hostname, Optional<String> ipv4Address, Optional<String> ipv6Address) { this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); - this.primary = primary; - this.secondary = secondary; + this.ipv4Address = Objects.requireNonNull(ipv4Address, "ipv4Address must be non-null"); + this.ipv6Address = Objects.requireNonNull(ipv6Address, "ipv6Address must be non-null"); } /** @@ -350,13 +375,17 @@ public class IP { * A successful allocation is guaranteed to have an IPv6 address, but may also have an IPv4 address if the * hostname of the IPv6 address has an A record. * - * @param ipAddress Unassigned IPv6 address + * @param ipv6Address Unassigned IPv6 address * @param resolver DNS name resolver to use * @throws IllegalArgumentException if DNS is misconfigured * @return An allocation containing 1 IPv6 address and 1 IPv4 address (if hostname is dual-stack) */ - private static Allocation ofIpv6(String ipAddress, NameResolver resolver) { - String hostname6 = resolver.resolveHostname(ipAddress).orElseThrow(() -> new IllegalArgumentException("Could not resolve IP address: " + ipAddress)); + private static Allocation ofIpv6(String ipv6Address, NameResolver resolver) { + if (!isV6(ipv6Address)) { + throw new IllegalArgumentException("Invalid IPv6 address '" + ipv6Address + "'"); + } + + String hostname6 = resolver.resolveHostname(ipv6Address).orElseThrow(() -> new IllegalArgumentException("Could not resolve IP address: " + ipv6Address)); List<String> ipv4Addresses = resolver.resolveAll(hostname6).stream() .filter(IP::isV4) .collect(Collectors.toList()); @@ -369,10 +398,10 @@ public class IP { if (!hostname6.equals(hostname4)) { throw new IllegalArgumentException(String.format("Hostnames resolved from each IP address do not " + "point to the same hostname [%s -> %s, %s -> %s]", - ipAddress, hostname6, addr, hostname4)); + ipv6Address, hostname6, addr, hostname4)); } }); - return new Allocation(hostname6, ipAddress, ipv4Address); + return new Allocation(hostname6, ipv4Address, Optional.of(ipv6Address)); } /** @@ -391,7 +420,11 @@ public class IP { throw new IllegalArgumentException("Hostname " + hostname4 + " did not resolve to exactly 1 address. " + "Resolved: " + addresses); } - return new Allocation(hostname4, addresses.get(0), Optional.empty()); + return new Allocation(hostname4, Optional.of(addresses.get(0)), Optional.empty()); + } + + private static Allocation ofAddress(Address address) { + return new Allocation(address.hostname(), Optional.empty(), Optional.empty()); } /** Hostname pointing to the IP addresses in this */ @@ -399,27 +432,28 @@ public class IP { return hostname; } - /** Primary address of this allocation */ - public String primary() { - return primary; + /** IPv4 address of this allocation */ + public Optional<String> ipv4Address() { + return ipv4Address; } - /** Secondary address of this allocation */ - public Optional<String> secondary() { - return secondary; + /** IPv6 address of this allocation */ + public Optional<String> ipv6Address() { + return ipv6Address; } /** All IP addresses in this */ public Set<String> addresses() { ImmutableSet.Builder<String> builder = ImmutableSet.builder(); - secondary.ifPresent(builder::add); - builder.add(primary); + ipv4Address.ifPresent(builder::add); + ipv6Address.ifPresent(builder::add); return builder.build(); } @Override public String toString() { - return String.format("IP allocation [primary=%s, secondary=%s]", primary, secondary.orElse("<none>")); + return String.format("Address allocation [hostname=%s, IPv4=%s, IPv6=%s]", + hostname, ipv4Address.orElse("<none>"), ipv6Address.orElse("<none>")); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index 2ddbd6def6f..3979b898145 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -47,6 +47,7 @@ public class ApplicationSerializer { private static final String groupsKey = "groups"; private static final String nodeResourcesKey = "resources"; private static final String scalingEventsKey = "scalingEvents"; + private static final String autoscalingStatusKey = "autoscalingStatus"; private static final String fromKey = "from"; private static final String toKey = "to"; private static final String generationKey = "generation"; @@ -95,6 +96,7 @@ public class ApplicationSerializer { cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject(suggestedResourcesKey))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject(targetResourcesKey))); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray(scalingEventsKey)); + clusterObject.setString(autoscalingStatusKey, cluster.autoscalingStatus()); } private static Cluster clusterFromSlime(String id, Inspector clusterObject) { @@ -104,7 +106,8 @@ public class ApplicationSerializer { clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)), optionalClusterResourcesFromSlime(clusterObject.field(suggestedResourcesKey)), optionalClusterResourcesFromSlime(clusterObject.field(targetResourcesKey)), - scalingEventsFromSlime(clusterObject.field(scalingEventsKey))); + scalingEventsFromSlime(clusterObject.field(scalingEventsKey)), + clusterObject.field(autoscalingStatusKey).asString()); } private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index b0baae650e4..6462fb6f19d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -71,47 +71,47 @@ public class GroupPreparer { } // There were some changes, so re-do the allocation with locks - try (Mutex lock = nodeRepository.lock(application)) { - try (Mutex allocationLock = nodeRepository.lockUnallocated()) { - NodeAllocation allocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, - highestIndex, wantedGroups, allocationLock); - - if (nodeRepository.zone().getCloud().dynamicProvisioning()) { - Version osVersion = nodeRepository.osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); - List<ProvisionedHost> provisionedHosts = allocation.getFulfilledDockerDeficit() - .map(deficit -> hostProvisioner.get().provisionHosts(nodeRepository.database().getProvisionIndexes(deficit.getCount()), - deficit.getFlavor(), - application, - osVersion, - requestedNodes.isExclusive() ? HostSharing.exclusive : HostSharing.any)) - .orElseGet(List::of); - - // At this point we have started provisioning of the hosts, the first priority is to make sure that - // the returned hosts are added to the node-repo so that they are tracked by the provision maintainers - List<Node> hosts = provisionedHosts.stream() - .map(ProvisionedHost::generateHost) - .collect(Collectors.toList()); - nodeRepository.addNodes(hosts, Agent.application); - - // Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit - List<NodeCandidate> candidates = provisionedHosts.stream() - .map(host -> NodeCandidate.createNewExclusiveChild(host.generateNode(), - host.generateHost())) - .collect(Collectors.toList()); - allocation.offer(candidates); - } - - if (! allocation.fulfilled() && requestedNodes.canFail()) - throw new OutOfCapacityException((cluster.group().isPresent() ? "Out of capacity on " + cluster.group().get() :"") + - allocation.outOfCapacityDetails()); - - // Carry out and return allocation - nodeRepository.reserve(allocation.reservableNodes()); - nodeRepository.addDockerNodes(new LockedNodeList(allocation.newNodes(), allocationLock)); - List<Node> acceptedNodes = allocation.finalNodes(); - surplusActiveNodes.removeAll(acceptedNodes); - return acceptedNodes; + try (Mutex lock = nodeRepository.lock(application); + Mutex allocationLock = nodeRepository.lockUnallocated()) { + + NodeAllocation allocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, + highestIndex, wantedGroups, allocationLock); + + if (nodeRepository.zone().getCloud().dynamicProvisioning()) { + Version osVersion = nodeRepository.osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); + List<ProvisionedHost> provisionedHosts = allocation.getFulfilledDockerDeficit() + .map(deficit -> hostProvisioner.get().provisionHosts(nodeRepository.database().getProvisionIndexes(deficit.getCount()), + deficit.getFlavor(), + application, + osVersion, + requestedNodes.isExclusive() ? HostSharing.exclusive : HostSharing.any)) + .orElseGet(List::of); + + // At this point we have started provisioning of the hosts, the first priority is to make sure that + // the returned hosts are added to the node-repo so that they are tracked by the provision maintainers + List<Node> hosts = provisionedHosts.stream() + .map(ProvisionedHost::generateHost) + .collect(Collectors.toList()); + nodeRepository.addNodes(hosts, Agent.application); + + // Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit + List<NodeCandidate> candidates = provisionedHosts.stream() + .map(host -> NodeCandidate.createNewExclusiveChild(host.generateNode(), + host.generateHost())) + .collect(Collectors.toList()); + allocation.offer(candidates); } + + if (! allocation.fulfilled() && requestedNodes.canFail()) + throw new OutOfCapacityException((cluster.group().isPresent() ? "Out of capacity on " + cluster.group().get() :"") + + allocation.outOfCapacityDetails()); + + // Carry out and return allocation + nodeRepository.reserve(allocation.reservableNodes()); + nodeRepository.addDockerNodes(new LockedNodeList(allocation.newNodes(), allocationLock)); + List<Node> acceptedNodes = allocation.finalNodes(); + surplusActiveNodes.removeAll(acceptedNodes); + return acceptedNodes; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java index 96053fdaa91..af3bde02421 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java @@ -82,7 +82,11 @@ public class HostCapacity { * Number of free (not allocated) IP addresses assigned to the dockerhost. */ int freeIPs(Node dockerHost) { - return dockerHost.ipConfig().pool().findUnused(allNodes).size(); + if (dockerHost.type() == NodeType.host) { + return dockerHost.ipConfig().pool().eventuallyUnusedAddressCount(allNodes); + } else { + return dockerHost.ipConfig().pool().findUnusedIpAddresses(allNodes).size(); + } } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index f8231072a28..14937e6afeb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -363,11 +363,11 @@ abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidate> { try { allocation = parent.get().ipConfig().pool().findAllocation(allNodes, nodeRepository.nameResolver()); if (allocation.isEmpty()) return new InvalidNodeCandidate(resources, freeParentCapacity, parent.get(), - "No IP addresses available on parent host"); + "No addresses available on parent host"); } catch (Exception e) { - log.warning("Failed allocating IP address on " + parent.get() +": " + Exceptions.toMessageString(e)); + log.warning("Failed allocating address on " + parent.get() +": " + Exceptions.toMessageString(e)); return new InvalidNodeCandidate(resources, freeParentCapacity, parent.get(), - "Failed when allocating IP address on host"); + "Failed when allocating address on host"); } Node node = Node.createDockerNode(allocation.get().addresses(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index edf151ff2d8..ede6f4ef250 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.AllocationOptimizer; import com.yahoo.vespa.hosted.provision.autoscale.Limits; import com.yahoo.vespa.hosted.provision.autoscale.ResourceTarget; +import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; @@ -132,7 +133,7 @@ public class NodeRepositoryProvisioner implements Provisioner { @Override public void remove(ApplicationTransaction transaction) { - nodeRepository.deactivate(transaction); + nodeRepository.remove(transaction); loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.deactivate(transaction)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 61cedbb9373..02621c79019 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -7,10 +7,12 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.OsVersion; import com.yahoo.vespa.hosted.provision.node.Status; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -26,25 +28,33 @@ public class ProvisionedHost { private final String hostHostname; private final Flavor hostFlavor; private final Optional<ApplicationId> exclusiveTo; - private final String nodeHostname; + private final List<Address> nodeAddresses; private final NodeResources nodeResources; private final Version osVersion; public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, Optional<ApplicationId> exclusiveTo, - String nodeHostname, NodeResources nodeResources, Version osVersion) { + List<Address> nodeAddresses, NodeResources nodeResources, Version osVersion) { this.id = Objects.requireNonNull(id, "Host id must be set"); this.hostHostname = Objects.requireNonNull(hostHostname, "Host hostname must be set"); this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); this.exclusiveTo = Objects.requireNonNull(exclusiveTo, "exclusiveTo must be set"); - this.nodeHostname = Objects.requireNonNull(nodeHostname, "Node hostname must be set"); + this.nodeAddresses = validateNodeAddresses(nodeAddresses); this.nodeResources = Objects.requireNonNull(nodeResources, "Node resources must be set"); this.osVersion = Objects.requireNonNull(osVersion, "OS version must be set"); } + private static List<Address> validateNodeAddresses(List<Address> nodeAddresses) { + Objects.requireNonNull(nodeAddresses, "Node addresses must be set"); + if (nodeAddresses.isEmpty()) { + throw new IllegalArgumentException("There must be at least one node address"); + } + return nodeAddresses; + } + /** Generate {@link Node} instance representing the provisioned physical host */ public Node generateHost() { Node.Builder builder = Node - .create(id, IP.Config.EMPTY, hostHostname, hostFlavor, NodeType.host) + .create(id, IP.Config.of(Set.of(), Set.of(), nodeAddresses), hostHostname, hostFlavor, NodeType.host) .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))); exclusiveTo.ifPresent(builder::exclusiveTo); return builder.build(); @@ -52,7 +62,7 @@ public class ProvisionedHost { /** Generate {@link Node} instance representing the node running on this physical host */ public Node generateNode() { - return Node.createDockerNode(Set.of(), nodeHostname, hostHostname, nodeResources, NodeType.tenant).build(); + return Node.createDockerNode(Set.of(), nodeHostname(), hostHostname, nodeResources, NodeType.tenant).build(); } public String getId() { @@ -68,7 +78,11 @@ public class ProvisionedHost { } public String nodeHostname() { - return nodeHostname; + return nodeAddresses.get(0).hostname(); + } + + public List<Address> nodeAddresses() { + return nodeAddresses; } public NodeResources nodeResources() { return nodeResources; } @@ -81,14 +95,14 @@ public class ProvisionedHost { return id.equals(that.id) && hostHostname.equals(that.hostHostname) && hostFlavor.equals(that.hostFlavor) && - nodeHostname.equals(that.nodeHostname) && + nodeAddresses.equals(that.nodeAddresses) && nodeResources.equals(that.nodeResources) && osVersion.equals(that.osVersion); } @Override public int hashCode() { - return Objects.hash(id, hostHostname, hostFlavor, nodeHostname, nodeResources, osVersion); + return Objects.hash(id, hostHostname, hostFlavor, nodeAddresses, nodeResources, osVersion); } @Override @@ -97,7 +111,7 @@ public class ProvisionedHost { "id='" + id + '\'' + ", hostHostname='" + hostHostname + '\'' + ", hostFlavor=" + hostFlavor + - ", nodeHostname='" + nodeHostname + '\'' + + ", nodeAddresses='" + nodeAddresses + '\'' + ", nodeResources=" + nodeResources + ", osVersion=" + osVersion + '}'; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index 9433b89ddc4..91b54fa37e9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import java.net.URI; @@ -51,6 +52,8 @@ public class ApplicationSerializer { toSlime(currentResources, clusterObject.setObject("current")); cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject("suggested"))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject("target"))); + scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); + clusterObject.setString("autoscalingStatus", cluster.autoscalingStatus()); } private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { @@ -59,4 +62,13 @@ public class ApplicationSerializer { NodeResourcesSerializer.toSlime(resources.nodeResources(), clusterResourcesObject.setObject("resources")); } + private static void scalingEventsToSlime(List<ScalingEvent> scalingEvents, Cursor scalingEventsArray) { + for (ScalingEvent scalingEvent : scalingEvents) { + Cursor scalingEventObject = scalingEventsArray.addObject(); + toSlime(scalingEvent.from(), scalingEventObject.setObject("from")); + toSlime(scalingEvent.to(), scalingEventObject.setObject("to")); + scalingEventObject.setLong("at", scalingEvent.at().toEpochMilli()); + } + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 304cebb3c01..c43629aeb09 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; @@ -256,8 +257,12 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { Set<String> ipAddressPool = new HashSet<>(); inspector.field("additionalIpAddresses").traverse((ArrayTraverser) (i, item) -> ipAddressPool.add(item.asString())); + List<Address> addressPool = new ArrayList<>(); + inspector.field("additionalHostnames").traverse((ArrayTraverser) (i, item) -> + addressPool.add(new Address(item.asString()))); + Node.Builder builder = Node.create(inspector.field("openStackId").asString(), - IP.Config.of(ipAddresses, ipAddressPool, List.of()), + IP.Config.of(ipAddresses, ipAddressPool, addressPool), inspector.field("hostname").asString(), flavorFromSlime(inspector), nodeTypeFromSlime(inspector.field("type"))); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 5813a7067cd..5393aa7cfb8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import org.junit.Test; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -44,11 +45,13 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 5, 1, hostResources); + tester.clock().advance(Duration.ofDays(1)); assertTrue("No measurements -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); tester.addCpuMeasurements(0.25f, 1f, 59, application1); assertTrue("Too few measurements -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); + tester.clock().advance(Duration.ofDays(1)); tester.addCpuMeasurements(0.25f, 1f, 60, application1); ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", 15, 1, 1.3, 28.6, 28.6, @@ -58,6 +61,8 @@ public class AutoscalingTest { assertTrue("Cluster in flux -> No further change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); tester.deactivateRetired(application1, cluster1, scaledResources); + + tester.clock().advance(Duration.ofDays(1)); tester.addCpuMeasurements(0.8f, 1f, 3, application1); assertTrue("Load change is large, but insufficient measurements for new config -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); @@ -112,6 +117,7 @@ public class AutoscalingTest { tester.nodeRepository().getNodes(application1).stream() .allMatch(n -> n.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.slow); + tester.clock().advance(Duration.ofDays(1)); tester.addCpuMeasurements(0.25f, 1f, 120, application1); // Changing min and max from slow to any ClusterResources min = new ClusterResources( 2, 1, @@ -184,7 +190,7 @@ public class AutoscalingTest { } @Test - public void test_autoscaling_limits_when_min_equals_xax() { + public void test_autoscaling_limits_when_min_equals_max() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = min; @@ -195,6 +201,7 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 5, 1, resources); + tester.clock().advance(Duration.ofDays(1)); tester.addCpuMeasurements(0.25f, 1f, 120, application1); assertTrue(tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); } @@ -283,6 +290,31 @@ public class AutoscalingTest { // deploy tester.deploy(application1, cluster1, 6, 1, hostResources.withVcpu(hostResources.vcpu() / 2)); + tester.clock().advance(Duration.ofDays(1)); + tester.addMemMeasurements(0.02f, 0.95f, 120, application1); + tester.assertResources("Scaling down", + 6, 1, 2.8, 4.0, 95.0, + tester.autoscale(application1, cluster1.id(), min, max).target()); + } + + @Test + public void scaling_down_only_after_delay() { + NodeResources hostResources = new NodeResources(6, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); + AutoscalingTester tester = new AutoscalingTester(hostResources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1"); + + tester.deploy(application1, cluster1, 6, 1, hostResources.withVcpu(hostResources.vcpu() / 2)); + + // No autoscaling as it is too soon to scale down after initial deploy (counting as a scaling event) + tester.addMemMeasurements(0.02f, 0.95f, 120, application1); + assertTrue(tester.autoscale(application1, cluster1.id(), min, max).target().isEmpty()); + + // Trying the same a day later causes autoscaling + tester.clock().advance(Duration.ofDays(1)); tester.addMemMeasurements(0.02f, 0.95f, 120, application1); tester.assertResources("Scaling down", 6, 1, 2.8, 4.0, 95.0, @@ -344,6 +376,7 @@ public class AutoscalingTest { // deploy (Why 103 Gb memory? See AutoscalingTester.MockHostResourcesCalculator tester.deploy(application1, cluster1, 5, 1, new NodeResources(3, 103, 100, 1)); + tester.clock().advance(Duration.ofDays(1)); tester.addMemMeasurements(0.9f, 0.6f, 120, application1); ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high.", 8, 1, 3, 83, 34.3, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 4d8b6d13a86..3faa4c244ee 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; @@ -294,7 +295,7 @@ class AutoscalingTester { "hostname" + index, hostFlavor, Optional.empty(), - "nodename" + index, + List.of(new Address("nodename" + index)), resources, osVersion)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 5e318e00288..4b14174488e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -110,9 +110,8 @@ public class AutoscalingMaintainerTest { assertEquals(firstMaintenanceTime.toEpochMilli(), tester.deployer().lastDeployTime(app1).get().toEpochMilli()); // Add measurement of the expected generation, leading to rescaling - tester.clock().advance(Duration.ofSeconds(1)); + tester.clock().advance(Duration.ofHours(2)); tester.addMeasurements(0.1f, 0.1f, 0.1f, 1, 500, app1); - //tester.clock().advance(Duration.ofSeconds(1)); Instant lastMaintenanceTime = tester.clock().instant(); tester.maintainer().maintain(); assertEquals(lastMaintenanceTime.toEpochMilli(), tester.deployer().lastDeployTime(app1).get().toEpochMilli()); @@ -122,10 +121,10 @@ public class AutoscalingMaintainerTest { @Test public void test_toString() { - assertEquals("4 * [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb] (total: [vcpu: 4.0, memory: 8.0 Gb, disk: 16.0 Gb])", + assertEquals("4 nodes with [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb, bandwidth: 1.0 Gbps] (total: [vcpu: 4.0, memory: 8.0 Gb, disk 16.0 Gb, bandwidth: 4.0 Gbps])", AutoscalingMaintainer.toString(new ClusterResources(4, 1, new NodeResources(1, 2, 4, 1)))); - assertEquals("4 (in 2 groups) * [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb] (total: [vcpu: 4.0, memory: 8.0 Gb, disk: 16.0 Gb])", + assertEquals("4 nodes (in 2 groups) with [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb, bandwidth: 1.0 Gbps] (total: [vcpu: 4.0, memory: 8.0 Gb, disk 16.0 Gb, bandwidth: 4.0 Gbps])", AutoscalingMaintainer.toString(new ClusterResources(4, 2, new NodeResources(1, 2, 4, 1)))); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 478376bc0cd..2833c4e11ba 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.custom.HostCapacity; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; @@ -208,12 +209,12 @@ public class DynamicProvisioningMaintainerTest { tester.maintainer.maintain(); assertTrue("No IP addresses written as DNS updates are failing", - provisioning.get().stream().allMatch(host -> host.ipConfig().pool().isEmpty())); + provisioning.get().stream().allMatch(host -> host.ipConfig().pool().getIpSet().isEmpty())); tester.hostProvisioner.without(Behaviour.failDnsUpdate); tester.maintainer.maintain(); assertTrue("IP addresses written as DNS updates are succeeding", - provisioning.get().stream().noneMatch(host -> host.ipConfig().pool().isEmpty())); + provisioning.get().stream().noneMatch(host -> host.ipConfig().pool().getIpSet().isEmpty())); } private static class DynamicProvisioningTester { @@ -338,7 +339,7 @@ public class DynamicProvisioningMaintainerTest { "hostname" + index, hostFlavor, Optional.empty(), - "nodename" + index, + List.of(new Address("nodename" + index)), resources, osVersion)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java index fb9c1ad0e5a..8101405ad7f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java @@ -86,8 +86,8 @@ public class IPTest { resolver.addReverseRecord("::2", "host1"); Optional<IP.Allocation> allocation = pool.findAllocation(emptyList, resolver); - assertEquals("::1", allocation.get().primary()); - assertFalse(allocation.get().secondary().isPresent()); + assertEquals(Optional.of("::1"), allocation.get().ipv6Address()); + assertFalse(allocation.get().ipv4Address().isPresent()); assertEquals("host3", allocation.get().hostname()); // Allocation fails if DNS record is missing @@ -105,16 +105,16 @@ public class IPTest { var pool = testPool(false); var allocation = pool.findAllocation(emptyList, resolver); assertFalse("Found allocation", allocation.isEmpty()); - assertEquals("127.0.0.1", allocation.get().primary()); - assertTrue("No secondary address", allocation.get().secondary().isEmpty()); + assertEquals(Optional.of("127.0.0.1"), allocation.get().ipv4Address()); + assertTrue("No IPv6 address", allocation.get().ipv6Address().isEmpty()); } @Test public void test_find_allocation_dual_stack() { IP.Pool pool = testPool(true); Optional<IP.Allocation> allocation = pool.findAllocation(emptyList, resolver); - assertEquals("::1", allocation.get().primary()); - assertEquals("127.0.0.2", allocation.get().secondary().get()); + assertEquals(Optional.of("::1"), allocation.get().ipv6Address()); + assertEquals("127.0.0.2", allocation.get().ipv4Address().get()); assertEquals("host3", allocation.get().hostname()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index 72f9e9597de..e63f31cf304 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -33,7 +33,8 @@ public class ApplicationSerializerTest { new ClusterResources(12, 6, new NodeResources(3, 6, 21, 24)), Optional.empty(), Optional.empty(), - List.of())); + List.of(), + "")); var minResources = new NodeResources(1, 2, 3, 4); clusters.add(new Cluster(ClusterSpec.Id.from("c2"), true, @@ -44,7 +45,8 @@ public class ApplicationSerializerTest { List.of(new ScalingEvent(new ClusterResources(10, 5, minResources), new ClusterResources(12, 6, minResources), 7L, - Instant.ofEpochMilli(12345L))))); + Instant.ofEpochMilli(12345L))), + "Autoscaling status")); Application original = new Application(ApplicationId.from("myTenant", "myApplication", "myInstance"), clusters); @@ -65,6 +67,7 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.suggestedResources(), serializedCluster.suggestedResources()); assertEquals(originalCluster.targetResources(), serializedCluster.targetResources()); assertEquals(originalCluster.scalingEvents(), serializedCluster.scalingEvents()); + assertEquals(originalCluster.autoscalingStatus(), serializedCluster.autoscalingStatus()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 4917a59879f..919d02c435c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -20,6 +20,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing; @@ -471,7 +472,7 @@ public class DynamicDockerProvisionTest { throw new OutOfCapacityException("No host flavor matches " + resources); return provisionIndexes.stream() .map(i -> new ProvisionedHost("id-" + i, "host-" + i, hostFlavor.get(), Optional.empty(), - "host-" + i + "-1", resources, osVersion)) + List.of(new Address("host-" + i + "-1")), resources, osVersion)) .collect(Collectors.toList()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java index c6e89680e85..808770f42dc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.IP; import org.junit.Before; import org.junit.Test; @@ -15,6 +16,8 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -32,8 +35,8 @@ public class HostCapacityTest { private HostCapacity capacity; private List<Node> nodes; private Node host1, host2, host3; - private final NodeResources resources1 = new NodeResources(1, 30, 20, 1.5); - private final NodeResources resources2 = new NodeResources(2, 40, 40, 0.5); + private final NodeResources dockerResources = new NodeResources(1, 30, 20, 1.5); + private final NodeResources docker2Resources = new NodeResources(2, 40, 40, 0.5); @Before public void setup() { @@ -48,15 +51,15 @@ public class HostCapacityTest { host3 = Node.create("host3", IP.Config.of(Set.of("::21"), generateIPs(22, 1), List.of()), "host3", nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); // Add two containers to host1 - var nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", resources1, NodeType.tenant).build(); - var nodeB = Node.createDockerNode(Set.of("::3"), "nodeB", "host1", resources1, NodeType.tenant).build(); + var nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", dockerResources, NodeType.tenant).build(); + var nodeB = Node.createDockerNode(Set.of("::3"), "nodeB", "host1", dockerResources, NodeType.tenant).build(); // Add two containers to host 2 (same as host 1) - var nodeC = Node.createDockerNode(Set.of("::12"), "nodeC", "host2", resources1, NodeType.tenant).build(); - var nodeD = Node.createDockerNode(Set.of("::13"), "nodeD", "host2", resources1, NodeType.tenant).build(); + var nodeC = Node.createDockerNode(Set.of("::12"), "nodeC", "host2", dockerResources, NodeType.tenant).build(); + var nodeD = Node.createDockerNode(Set.of("::13"), "nodeD", "host2", dockerResources, NodeType.tenant).build(); // Add a larger container to host3 - var nodeE = Node.createDockerNode(Set.of("::22"), "nodeE", "host3", resources2, NodeType.tenant).build(); + var nodeE = Node.createDockerNode(Set.of("::22"), "nodeE", "host3", docker2Resources, NodeType.tenant).build(); // init docker host capacity nodes = new ArrayList<>(List.of(host1, host2, host3, nodeA, nodeB, nodeC, nodeD, nodeE)); @@ -65,19 +68,19 @@ public class HostCapacityTest { @Test public void hasCapacity() { - assertTrue(capacity.hasCapacity(host1, resources1)); - assertTrue(capacity.hasCapacity(host1, resources2)); - assertTrue(capacity.hasCapacity(host2, resources1)); - assertTrue(capacity.hasCapacity(host2, resources2)); - assertFalse(capacity.hasCapacity(host3, resources1)); // No ip available - assertFalse(capacity.hasCapacity(host3, resources2)); // No ip available + assertTrue(capacity.hasCapacity(host1, dockerResources)); + assertTrue(capacity.hasCapacity(host1, docker2Resources)); + assertTrue(capacity.hasCapacity(host2, dockerResources)); + assertTrue(capacity.hasCapacity(host2, docker2Resources)); + assertFalse(capacity.hasCapacity(host3, dockerResources)); // No ip available + assertFalse(capacity.hasCapacity(host3, docker2Resources)); // No ip available // Add a new node to host1 to deplete the memory resource - Node nodeF = Node.createDockerNode(Set.of("::6"), "nodeF", "host1", resources1, NodeType.tenant).build(); + Node nodeF = Node.createDockerNode(Set.of("::6"), "nodeF", "host1", dockerResources, NodeType.tenant).build(); nodes.add(nodeF); capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); - assertFalse(capacity.hasCapacity(host1, resources1)); - assertFalse(capacity.hasCapacity(host1, resources2)); + assertFalse(capacity.hasCapacity(host1, dockerResources)); + assertFalse(capacity.hasCapacity(host1, docker2Resources)); } @Test @@ -112,19 +115,78 @@ public class HostCapacityTest { var nodeFlavors = FlavorConfigBuilder.createDummies("devhost", "container"); var devHost = Node.create("devhost", new IP.Config(Set.of("::1"), generateIPs(2, 10)), "devhost", nodeFlavors.getFlavorOrThrow("devhost"), NodeType.devhost).build(); - var cfg = Node.createDockerNode(Set.of("::2"), "cfg", "devhost", resources1, NodeType.config).build(); + var cfg = Node.createDockerNode(Set.of("::2"), "cfg", "devhost", dockerResources, NodeType.config).build(); var nodes = new ArrayList<>(List.of(cfg)); var capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); - assertTrue(capacity.hasCapacity(devHost, resources1)); + assertTrue(capacity.hasCapacity(devHost, dockerResources)); - var container1 = Node.createDockerNode(Set.of("::3"), "container1", "devhost", resources1, NodeType.tenant).build(); + var container1 = Node.createDockerNode(Set.of("::3"), "container1", "devhost", dockerResources, NodeType.tenant).build(); nodes = new ArrayList<>(List.of(cfg, container1)); capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); - assertFalse(capacity.hasCapacity(devHost, resources1)); + assertFalse(capacity.hasCapacity(devHost, dockerResources)); } + @Test + public void verifyCapacityFromAddresses() { + Node nodeA = Node.createDockerNode(Set.of("::2"), "nodeA", "host1", dockerResources, NodeType.tenant).build(); + Node nodeB = Node.createDockerNode(Set.of("::3"), "nodeB", "host1", dockerResources, NodeType.tenant).build(); + Node nodeC = Node.createDockerNode(Set.of("::4"), "nodeC", "host1", dockerResources, NodeType.tenant).build(); + + // host1 is a host with resources = 7-100-120-5 (7 vcpus, 100G memory, 120G disk, and 5Gbps), + // while nodeA-C have resources = dockerResources = 1-30-20-1.5 + + Node host1 = setupHostWithAdditionalHostnames("host1", "nodeA"); + // Allocating nodeA should be OK + assertTrue(hasCapacity(dockerResources, host1)); + // then, the second node lacks hostname address + assertFalse(hasCapacity(dockerResources, host1, nodeA)); + + host1 = setupHostWithAdditionalHostnames("host1", "nodeA", "nodeB"); + // Allocating nodeA and nodeB should be OK + assertTrue(hasCapacity(dockerResources, host1)); + assertTrue(hasCapacity(dockerResources, host1, nodeA)); + // but the third node lacks hostname address + assertFalse(hasCapacity(dockerResources, host1, nodeA, nodeB)); + + host1 = setupHostWithAdditionalHostnames("host1", "nodeA", "nodeB", "nodeC"); + // Allocating nodeA, nodeB, and nodeC should be OK + assertTrue(hasCapacity(dockerResources, host1)); + assertTrue(hasCapacity(dockerResources, host1, nodeA)); + assertTrue(hasCapacity(dockerResources, host1, nodeA, nodeB)); + // but the fourth node lacks hostname address + assertFalse(hasCapacity(dockerResources, host1, nodeA, nodeB, nodeC)); + + host1 = setupHostWithAdditionalHostnames("host1", "nodeA", "nodeB", "nodeC", "nodeD"); + // Allocating nodeA, nodeB, and nodeC should be OK + assertTrue(hasCapacity(dockerResources, host1)); + assertTrue(hasCapacity(dockerResources, host1, nodeA)); + assertTrue(hasCapacity(dockerResources, host1, nodeA, nodeB)); + // but the fourth lacks memory (host has 100G, while 4x30G = 120G + assertFalse(hasCapacity(dockerResources, host1, nodeA, nodeB, nodeC)); + } + + private Node setupHostWithAdditionalHostnames(String hostHostname, String... additionalHostnames) { + List<Address> addresses = Stream.of(additionalHostnames).map(Address::new).collect(Collectors.toList()); + + doAnswer(invocation -> ((Flavor)invocation.getArguments()[0]).resources()) + .when(hostResourcesCalculator).advertisedResourcesOf(any()); + + NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies( + "host", // 7-100-120-5 + "docker"); // 2- 40- 40-0.5 = docker2Resources + + return Node.create(hostHostname, IP.Config.of(Set.of("::1"), Set.of(), addresses), hostHostname, + nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); + } + + private boolean hasCapacity(NodeResources requestedCapacity, Node host, Node... remainingNodes) { + List<Node> nodes = Stream.concat(Stream.of(host), Stream.of(remainingNodes)).collect(Collectors.toList()); + var capacity = new HostCapacity(new LockedNodeList(nodes, () -> {}), hostResourcesCalculator); + return capacity.hasCapacity(host, requestedCapacity); + } + private Set<String> generateIPs(int start, int count) { // Allow 4 containers Set<String> ipAddressPool = new LinkedHashSet<>(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 2fe39780cf5..cbac5a39e09 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -103,9 +103,13 @@ public class ProvisioningTest { assertEquals(5, tester.getNodes(application1, Node.State.inactive).size()); // delete app + NodeList previouslyActive = tester.getNodes(application1, Node.State.active); + NodeList previouslyInactive = tester.getNodes(application1, Node.State.inactive); tester.remove(application1); - assertEquals(tester.toHostNames(state1.allHosts), tester.toHostNames(tester.nodeRepository().getNodes(application1, Node.State.inactive))); + assertEquals(tester.toHostNames(previouslyActive.asList()), tester.toHostNames(tester.nodeRepository().getNodes(application1, Node.State.inactive))); + assertEquals(tester.toHostNames(previouslyInactive.asList()), tester.toHostNames(tester.nodeRepository().getNodes(Node.State.dirty))); assertEquals(0, tester.getNodes(application1, Node.State.active).size()); + assertTrue(tester.nodeRepository().applications().get(application1).isEmpty()); // other application is unaffected assertEquals(state1App2.hostNames(), tester.toHostNames(tester.nodeRepository().getNodes(application2, Node.State.active))); @@ -121,6 +125,7 @@ public class ProvisioningTest { tester.activate(application2, state2App2.allHosts); // deploy first app again + tester.nodeRepository().setReady(tester.nodeRepository().getNodes(Node.State.dirty), Agent.system, "recycled"); SystemState state7 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); state7.assertEquals(state1); tester.activate(application1, state7.allHosts); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index d39ea3786f1..f012f0a428f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -242,8 +242,8 @@ public class ProvisioningTester { public void deactivate(ApplicationId applicationId) { try (var lock = nodeRepository.lock(applicationId)) { NestedTransaction deactivateTransaction = new NestedTransaction(); - nodeRepository.deactivate(new ApplicationTransaction(new ProvisionLock(applicationId, lock), - deactivateTransaction)); + nodeRepository.remove(new ApplicationTransaction(new ProvisionLock(applicationId, lock), + deactivateTransaction)); deactivateTransaction.commit(); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index a98d383e219..86427fe30ae 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -91,8 +91,9 @@ public class NodesV2ApiTest { // POST new nodes assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.8.1") + "," + // test with only 1 ip address - asHostJson("host9.yahoo.com", "large-variant", "127.0.9.1", "::9:1") + "," + - asNodeJson("parent2.yahoo.com", NodeType.host, "large-variant", Optional.of(TenantName.from("myTenant")), Optional.of(ApplicationId.from("tenant1", "app1", "instance1")), Optional.empty(), "127.0.127.1", "::127:1") + "," + + asHostJson("host9.yahoo.com", "large-variant", List.of("node9-1.yahoo.com"), "127.0.9.1", "::9:1") + "," + + asNodeJson("parent2.yahoo.com", NodeType.host, "large-variant", Optional.of(TenantName.from("myTenant")), + Optional.of(ApplicationId.from("tenant1", "app1", "instance1")), Optional.empty(), List.of(), "127.0.127.1", "::127:1") + "," + asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com", "::11") + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST), @@ -322,7 +323,7 @@ public class NodesV2ApiTest { // Attempt to POST host node with already assigned IP tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asHostJson("host200.yahoo.com", "default", "127.0.2.1") + "]", + "[" + asHostJson("host200.yahoo.com", "default", List.of(), "127.0.2.1") + "]", Request.Method.POST), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.2.1] to host200.yahoo.com: [127.0.2.1] already assigned to host2.yahoo.com\"}"); @@ -334,7 +335,7 @@ public class NodesV2ApiTest { // Node types running a single container can share their IP address with child node tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), Optional.empty(), "127.0.42.1") + "]", + "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), Optional.empty(), List.of(), "127.0.42.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", @@ -350,7 +351,7 @@ public class NodesV2ApiTest { // ... nor with child node on different host tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), Optional.empty(), "127.0.43.1") + "]", + "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), Optional.empty(), Optional.empty(), List.of(), "127.0.43.1") + "]", Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", @@ -392,7 +393,7 @@ public class NodesV2ApiTest { @Test public void fails_to_ready_node_with_hard_fail() throws Exception { assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asHostJson("host12.yahoo.com", "default") + "]"). + ("[" + asHostJson("host12.yahoo.com", "default", List.of()) + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); @@ -961,7 +962,8 @@ public class NodesV2ApiTest { public void test_node_switch_hostname() throws Exception { String hostname = "host42.yahoo.com"; // Add host with switch hostname - String json = asNodeJson(hostname, NodeType.host, "default", Optional.empty(), Optional.empty(), Optional.of("switch0"), "127.0.42.1", "::42:1"); + String json = asNodeJson(hostname, NodeType.host, "default", Optional.empty(), Optional.empty(), + Optional.of("switch0"), List.of(), "127.0.42.1", "::42:1"); assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + json + "]").getBytes(StandardCharsets.UTF_8), Request.Method.POST), @@ -1013,17 +1015,22 @@ public class NodesV2ApiTest { "\"flavor\":\"" + flavor + "\"}"; } - private static String asHostJson(String hostname, String flavor, String... ipAddress) { - return asNodeJson(hostname, NodeType.host, flavor, Optional.empty(), Optional.empty(), Optional.empty(), ipAddress); + private static String asHostJson(String hostname, String flavor, List<String> additionalHostnames, String... ipAddress) { + return asNodeJson(hostname, NodeType.host, flavor, Optional.empty(), Optional.empty(), Optional.empty(), + additionalHostnames, ipAddress); } - private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, Optional<String> switchHostname, String... ipAddress) { + private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional<TenantName> reservedTo, + Optional<ApplicationId> exclusiveTo, Optional<String> switchHostname, + List<String> additionalHostnames, String... ipAddress) { return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\"," + createIpAddresses(ipAddress) + "\"flavor\":\"" + flavor + "\"" + (reservedTo.map(tenantName -> ", \"reservedTo\":\"" + tenantName.value() + "\"").orElse("")) + (exclusiveTo.map(appId -> ", \"exclusiveTo\":\"" + appId.serializedForm() + "\"").orElse("")) + (switchHostname.map(s -> ", \"switchHostname\":\"" + s + "\"").orElse("")) + + (additionalHostnames.isEmpty() ? "" : ", \"additionalHostnames\":[\"" + + String.join("\",\"", additionalHostnames) + "\"]") + ", \"type\":\"" + nodeType + "\"}"; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json index 456ed18334e..82f7e04f92b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json @@ -62,7 +62,37 @@ "diskSpeed" : "fast", "storageType" : "any" } - } + }, + "scalingEvents" : [ + { + "from": { + "nodes": 0, + "groups": 0, + "resources": { + "vcpu" : 0.0, + "memoryGb": 0.0, + "diskGb": 0.0, + "bandwidthGbps": 0.0, + "diskSpeed": "fast", + "storageType": "any" + } + }, + "to": { + "nodes": 2, + "groups": 1, + "resources" : { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "fast", + "storageType": "local" + } + }, + "at" : 123 + } + ], + "autoscalingStatus" : "" } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json index bd22087ecfa..0ee590f60e0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json @@ -38,7 +38,37 @@ "diskSpeed": "fast", "storageType": "local" } - } + }, + "scalingEvents" : [ + { + "from": { + "nodes": 0, + "groups": 0, + "resources": { + "vcpu" : 0.0, + "memoryGb": 0.0, + "diskGb": 0.0, + "bandwidthGbps": 0.0, + "diskSpeed": "fast", + "storageType": "any" + } + }, + "to": { + "nodes": 2, + "groups": 1, + "resources" : { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "fast", + "storageType": "local" + } + }, + "at" : 123 + } + ], + "autoscalingStatus" : "" } } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json index dac9fd30267..809e58bd7b6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json @@ -25,5 +25,6 @@ "127.0.9.1", "::9:1" ], - "additionalIpAddresses": [] + "additionalIpAddresses": [], + "additionalHostnames": ["node9-1.yahoo.com"] } diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h index 822a4e4e028..8df7f29590b 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchcommon/src/vespa/searchcommon/attribute/config.h @@ -35,7 +35,7 @@ public: bool fastSearch() const { return _fastSearch; } bool huge() const { return _huge; } const PredicateParams &predicateParams() const { return _predicateParams; } - vespalib::eval::ValueType tensorType() const { return _tensorType; } + const vespalib::eval::ValueType & tensorType() const { return _tensorType; } DistanceMetric distance_metric() const { return _distance_metric; } const std::optional<HnswIndexParams>& hnsw_index_params() const { return _hnsw_index_params; } diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index b875ab8e058..2b2b8acbc50 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -179,23 +179,20 @@ public: { } - void notifyPutDone(IDestructorCallbackSP, document::GlobalId gid, uint32_t lid, SerialNum) override { + void notifyPut(IDestructorCallbackSP, document::GlobalId gid, uint32_t lid, SerialNum) override { _changeGid = gid; _changeLid = lid; _gidToLid[gid] = lid; ++_changes; } - void notifyRemove(IDestructorCallbackSP, document::GlobalId gid, SerialNum) override { + void notifyRemove(IDestructorCallbackSP, document::GlobalId gid, SerialNum) override { _changeGid = gid; _changeLid = 0; _gidToLid[gid] = 0; ++_changes; } - void notifyRemoveDone(document::GlobalId, SerialNum) override { - } - void assertChanges(document::GlobalId expGid, uint32_t expLid, uint32_t expChanges) { EXPECT_EQUAL(expGid, _changeGid); EXPECT_EQUAL(expLid, _changeLid); diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp index a10d48ee7fe..9d72045c918 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchcore/proton/server/executor_thread_service.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h> +#include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> #include <vespa/searchcore/proton/reference/gid_to_lid_change_handler.h> #include <vespa/searchlib/common/gatecallback.h> #include <map> @@ -112,11 +113,13 @@ public: struct Fixture { std::vector<std::shared_ptr<ListenerStats>> _statss; - std::shared_ptr<GidToLidChangeHandler> _handler; + std::shared_ptr<GidToLidChangeHandler> _real_handler; + std::shared_ptr<IGidToLidChangeHandler> _handler; Fixture() : _statss(), - _handler(std::make_shared<GidToLidChangeHandler>()) + _real_handler(std::make_shared<GidToLidChangeHandler>()), + _handler(_real_handler) { } @@ -127,7 +130,7 @@ struct Fixture void close() { - _handler->close(); + _real_handler->close(); } ListenerStats &addStats() { @@ -139,10 +142,15 @@ struct Fixture _handler->addListener(std::move(listener)); } - void notifyPutDone(GlobalId gid, uint32_t lid, SerialNum serialNum) { - vespalib::Gate gate; - _handler->notifyPutDone(std::make_shared<search::GateCallback>(gate), gid, lid, serialNum); - gate.await(); + void commit() { + auto pending = _handler->grab_pending_changes(); + if (pending) { + pending->notify_done(); + } + } + + void notifyPut(GlobalId gid, uint32_t lid, SerialNum serial_num) { + _handler->notifyPut(std::shared_ptr<search::IDestructorCallback>(), gid, lid, serial_num); } void notifyRemove(GlobalId gid, SerialNum serialNum) { @@ -151,10 +159,6 @@ struct Fixture gate.await(); } - void notifyRemoveDone(GlobalId gid, SerialNum serialNum) { - _handler->notifyRemoveDone(gid, serialNum); - } - void removeListeners(const vespalib::string &docTypeName, const std::set<vespalib::string> &keepNames) { _handler->removeListeners(docTypeName, keepNames); @@ -169,7 +173,8 @@ TEST_F("Test that we can register a listener", Fixture) TEST_DO(stats.assertListeners(1, 0, 0)); f.addListener(std::move(listener)); TEST_DO(stats.assertListeners(1, 1, 0)); - f.notifyPutDone(toGid(doc1), 10, 10); + f.notifyPut(toGid(doc1), 10, 10); + f.commit(); TEST_DO(stats.assertChanges(1, 0)); f.removeListeners("testdoc", {}); TEST_DO(stats.assertListeners(1, 1, 1)); @@ -192,7 +197,8 @@ TEST_F("Test that we can register multiple listeners", Fixture) TEST_DO(stats1.assertListeners(1, 1, 0)); TEST_DO(stats2.assertListeners(1, 1, 0)); TEST_DO(stats3.assertListeners(1, 1, 0)); - f.notifyPutDone(toGid(doc1), 10, 10); + f.notifyPut(toGid(doc1), 10, 10); + f.commit(); TEST_DO(stats1.assertChanges(1, 0)); TEST_DO(stats2.assertChanges(1, 0)); TEST_DO(stats3.assertChanges(1, 0)); @@ -250,62 +256,39 @@ public: } }; -TEST_F("Test that put is ignored if we have a pending remove", StatsFixture) +TEST_F("Test that multiple puts are processed", StatsFixture) { - f.notifyRemove(toGid(doc1), 20); - TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 10, 10); - TEST_DO(f.assertChanges(0, 1)); - f.notifyRemoveDone(toGid(doc1), 20); - TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 11, 30); - TEST_DO(f.assertChanges(1, 1)); + f.notifyPut(toGid(doc1), 10, 10); + TEST_DO(f.assertChanges(0, 0)); + f.notifyPut(toGid(doc1), 11, 20); + TEST_DO(f.assertChanges(0, 0)); + f.commit(); + TEST_DO(f.assertChanges(2, 0)); } -TEST_F("Test that pending removes are merged", StatsFixture) +TEST_F("Test that put is ignored if we have a pending remove", StatsFixture) { + f.notifyPut(toGid(doc1), 10, 10); + TEST_DO(f.assertChanges(0, 0)); f.notifyRemove(toGid(doc1), 20); TEST_DO(f.assertChanges(0, 1)); - f.notifyRemove(toGid(doc1), 40); + f.commit(); TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 10, 10); - TEST_DO(f.assertChanges(0, 1)); - f.notifyRemoveDone(toGid(doc1), 20); - TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 11, 30); - TEST_DO(f.assertChanges(0, 1)); - f.notifyRemoveDone(toGid(doc1), 40); - TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 12, 50); + f.notifyPut(toGid(doc1), 11, 30); + f.commit(); TEST_DO(f.assertChanges(1, 1)); } -TEST_F("Test that out of order notifyRemoveDone is handled", StatsFixture) +TEST_F("Test that pending removes are merged", StatsFixture) { - f.notifyRemove(toGid(doc1), 20); + f.notifyPut(toGid(doc1), 10, 10); + TEST_DO(f.assertChanges(0, 0)); + f.notifyRemove(toGid(doc1), 20); TEST_DO(f.assertChanges(0, 1)); f.notifyRemove(toGid(doc1), 40); TEST_DO(f.assertChanges(0, 1)); - f.notifyRemoveDone(toGid(doc1), 40); - TEST_DO(f.assertChanges(0, 1)); - f.notifyRemoveDone(toGid(doc1), 20); + f.commit(); TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 12, 50); - TEST_DO(f.assertChanges(1, 1)); -} - -TEST_F("Test that out of order notifyPutDone is partially handled", StatsFixture) -{ - f.notifyRemove(toGid(doc1), 20); - TEST_DO(f.assertChanges(0, 1)); - f.notifyPutDone(toGid(doc1), 12, 50); - TEST_DO(f.assertChanges(1, 1)); - f.notifyPutDone(toGid(doc1), 11, 40); - TEST_DO(f.assertChanges(1, 1)); - f.notifyPutDone(toGid(doc1), 13, 55); - TEST_DO(f.assertChanges(2, 1)); - f.notifyRemoveDone(toGid(doc1), 20); - TEST_DO(f.assertChanges(2, 1)); } } diff --git a/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt index a98b095cc21..1ba96e9adf5 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt @@ -10,7 +10,7 @@ vespa_add_library(searchcore_reference STATIC gid_to_lid_change_registrator.cpp gid_to_lid_mapper.cpp gid_to_lid_mapper_factory.cpp - pending_notify_remove_done.cpp + pending_gid_to_lid_changes.cpp DEPENDS searchcore_attribute searchcore_documentmetastore diff --git a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp index 6c45096a53f..a579a11e61f 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dummy_gid_to_lid_change_handler.h" +#include "i_pending_gid_to_lid_changes.h" namespace proton { @@ -9,7 +10,7 @@ DummyGidToLidChangeHandler::DummyGidToLidChangeHandler() = default; DummyGidToLidChangeHandler::~DummyGidToLidChangeHandler() = default; void -DummyGidToLidChangeHandler::notifyPutDone(IDestructorCallbackSP , GlobalId, uint32_t, SerialNum) +DummyGidToLidChangeHandler::notifyPut(IDestructorCallbackSP, GlobalId, uint32_t, SerialNum) { } @@ -18,9 +19,10 @@ DummyGidToLidChangeHandler::notifyRemove(IDestructorCallbackSP , GlobalId, Seria { } -void -DummyGidToLidChangeHandler::notifyRemoveDone(GlobalId, SerialNum) +std::unique_ptr<IPendingGidToLidChanges> +DummyGidToLidChangeHandler::grab_pending_changes() { + return {}; } void diff --git a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h index d5f6d788885..54cc0e2144a 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h @@ -22,11 +22,11 @@ public: DummyGidToLidChangeHandler(); ~DummyGidToLidChangeHandler() override; - void notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serialNum) override; + void notifyPut(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serial_num) override; void notifyRemove(IDestructorCallbackSP context, GlobalId gid, SerialNum serialNum) override; - void notifyRemoveDone(GlobalId gid, SerialNum serialNum) override; void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override; void removeListeners(const vespalib::string &docTypeName, const std::set<vespalib::string> &keepNames) override; + std::unique_ptr<IPendingGidToLidChanges> grab_pending_changes() override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp index 0c9087405b6..805f19b5bf0 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp @@ -2,6 +2,7 @@ #include "gid_to_lid_change_handler.h" #include "i_gid_to_lid_change_listener.h" +#include "pending_gid_to_lid_changes.h" #include <vespa/vespalib/util/lambdatask.h> #include <cassert> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -14,8 +15,8 @@ GidToLidChangeHandler::GidToLidChangeHandler() : _lock(), _listeners(), _closed(false), - _pendingRemove() - + _pendingRemove(), + _pending_changes() { } @@ -43,6 +44,13 @@ GidToLidChangeHandler::notifyRemove(IDestructorCallbackSP context, GlobalId gid) } void +GidToLidChangeHandler::notifyPut(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serial_num) +{ + lock_guard guard(_lock); + _pending_changes.emplace_back(std::move(context), gid, lid, serial_num, false); +} + +void GidToLidChangeHandler::notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serialNum) { lock_guard guard(_lock); @@ -79,6 +87,7 @@ GidToLidChangeHandler::notifyRemove(IDestructorCallbackSP context, GlobalId gid, } else { notifyRemove(std::move(context), gid); } + _pending_changes.emplace_back(IDestructorCallbackSP(), gid, 0, serialNum, true); } void @@ -96,6 +105,16 @@ GidToLidChangeHandler::notifyRemoveDone(GlobalId gid, SerialNum serialNum) } } +std::unique_ptr<IPendingGidToLidChanges> +GidToLidChangeHandler::grab_pending_changes() +{ + lock_guard guard(_lock); + if (_pending_changes.empty()) { + return {}; + } + return std::make_unique<PendingGidToLidChanges>(*this, std::move(_pending_changes)); +} + void GidToLidChangeHandler::close() { diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h index 13a51edd0b5..25645aebcc9 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h @@ -3,8 +3,8 @@ #pragma once #include "i_gid_to_lid_change_handler.h" +#include "pending_gid_to_lid_change.h" #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/document/base/globalid.h> #include <vector> #include <mutex> @@ -44,6 +44,7 @@ class GidToLidChangeHandler : public std::enable_shared_from_this<GidToLidChange Listeners _listeners; bool _closed; vespalib::hash_map<GlobalId, PendingRemoveEntry, GlobalId::hash> _pendingRemove; + std::vector<PendingGidToLidChange> _pending_changes; void notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid); void notifyRemove(IDestructorCallbackSP context, GlobalId gid); @@ -51,9 +52,11 @@ public: GidToLidChangeHandler(); ~GidToLidChangeHandler() override; - void notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serialNum) override; + void notifyPut(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serial_num) override; + void notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serialNum); void notifyRemove(IDestructorCallbackSP context, GlobalId gid, SerialNum serialNum) override; - void notifyRemoveDone(GlobalId gid, SerialNum serialNum) override; + void notifyRemoveDone(GlobalId gid, SerialNum serialNum); + std::unique_ptr<IPendingGidToLidChanges> grab_pending_changes() override; /** * Close handler, further notifications are blocked. diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h index cbafef57e46..0dad58b31ae 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h @@ -10,6 +10,8 @@ namespace document { class GlobalId; } namespace proton { +class IPendingGidToLidChanges; + /* * Interface class for registering listeners that get notification when * gid to lid mapping changes. @@ -36,11 +38,15 @@ public: virtual void removeListeners(const vespalib::string &docTypeName, const std::set<vespalib::string> &keepNames) = 0; /** - * Notify gid to lid mapping change. + * Notify pending gid to lid mapping change. Passed on to listeners later + * when force commit has made changes visible. + */ + virtual void notifyPut(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serial_num) = 0; + /** + * Notify removal of gid. Passed on to listeners at once. */ - virtual void notifyPutDone(IDestructorCallbackSP context, GlobalId gid, uint32_t lid, SerialNum serialNum) = 0; virtual void notifyRemove(IDestructorCallbackSP context, GlobalId gid, SerialNum serialNum) = 0; - virtual void notifyRemoveDone(GlobalId gid, SerialNum serialNum) = 0; + virtual std::unique_ptr<IPendingGidToLidChanges> grab_pending_changes() = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h b/searchcore/src/vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h new file mode 100644 index 00000000000..6e4e9f240ff --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h @@ -0,0 +1,18 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace proton { + +/* + * Interface class for a container of gid to lid changes awaiting a + * force commit. + */ +class IPendingGidToLidChanges +{ +public: + virtual ~IPendingGidToLidChanges() = default; + virtual void notify_done() = 0; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_change.h b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_change.h new file mode 100644 index 00000000000..1577077a47b --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_change.h @@ -0,0 +1,44 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/common/idestructorcallback.h> +#include <vespa/document/base/globalid.h> +#include <vespa/searchlib/common/serialnum.h> + +namespace proton { + +/* + * Class for a gid to lid change awaiting a force commit. + */ +class PendingGidToLidChange +{ + using Context = std::shared_ptr<search::IDestructorCallback>; + using GlobalId = document::GlobalId; + using SerialNum = search::SerialNum; + + Context _context; + SerialNum _serial_num; + GlobalId _gid; + uint32_t _lid; + bool _is_remove; +public: + PendingGidToLidChange(); + PendingGidToLidChange(Context context, const GlobalId& gid, uint32_t lid, SerialNum serial_num, bool is_remove_) noexcept + : _context(std::move(context)), + _serial_num(serial_num), + _gid(gid), + _lid(lid), + _is_remove(is_remove_) + { + } + ~PendingGidToLidChange() = default; + + Context steal_context() && { return std::move(_context); } + const GlobalId &get_gid() const { return _gid; } + uint32_t get_lid() const { return _lid; } + SerialNum get_serial_num() const { return _serial_num; } + bool is_remove() const { return _is_remove; } +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.cpp b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.cpp new file mode 100644 index 00000000000..bb9da2d9c7f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.cpp @@ -0,0 +1,29 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "pending_gid_to_lid_changes.h" +#include "gid_to_lid_change_handler.h" + +namespace proton { + +PendingGidToLidChanges::PendingGidToLidChanges(GidToLidChangeHandler& handler, std::vector<PendingGidToLidChange> &&pending_changes) + : IPendingGidToLidChanges(), + _handler(handler), + _pending_changes(std::move(pending_changes)) +{ +} + +PendingGidToLidChanges::~PendingGidToLidChanges() = default; + +void +PendingGidToLidChanges::notify_done() +{ + for (auto& change : _pending_changes) { + if (change.is_remove()) { + _handler.notifyRemoveDone(change.get_gid(), change.get_serial_num()); + } else { + _handler.notifyPutDone(std::move(change).steal_context(), change.get_gid(), change.get_lid(), change.get_serial_num()); + } + } +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.h b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.h new file mode 100644 index 00000000000..a3e74e74b1f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/pending_gid_to_lid_changes.h @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_pending_gid_to_lid_changes.h" +#include "pending_gid_to_lid_change.h" +#include <vector> + +namespace proton { + +class GidToLidChangeHandler; + +/* + * Class for a vector of gid to lid changes awaiting a force commit. + */ +class PendingGidToLidChanges : public IPendingGidToLidChanges +{ + GidToLidChangeHandler& _handler; + std::vector<PendingGidToLidChange> _pending_changes; +public: + PendingGidToLidChanges(GidToLidChangeHandler& handler, std::vector<PendingGidToLidChange> &&pending_changes); + ~PendingGidToLidChanges() override; + void notify_done() override; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.cpp b/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.cpp deleted file mode 100644 index e806628bc02..00000000000 --- a/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "pending_notify_remove_done.h" -#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> -#include <cassert> - -namespace proton -{ - -PendingNotifyRemoveDone::PendingNotifyRemoveDone() - : _gidToLidChangeHandler(nullptr), - _gid(), - _serialNum(0), - _pending(false) -{ -} - -PendingNotifyRemoveDone::PendingNotifyRemoveDone(PendingNotifyRemoveDone &&rhs) - : _gidToLidChangeHandler(rhs._gidToLidChangeHandler), - _gid(rhs._gid), - _serialNum(rhs._serialNum), - _pending(rhs._pending) -{ - rhs._pending = false; -} - -PendingNotifyRemoveDone::~PendingNotifyRemoveDone() -{ - assert(!_pending); // Fail if notifyRemoveDone is still pending -} - -void -PendingNotifyRemoveDone::setup(IGidToLidChangeHandler &gidToLidChangeHandler, document::GlobalId gid, search::SerialNum serialNum) -{ - _gidToLidChangeHandler = &gidToLidChangeHandler; - _gid = gid; - _serialNum = serialNum; - _pending = true; -} - -void -PendingNotifyRemoveDone::invoke() -{ - if (_pending) { - _gidToLidChangeHandler->notifyRemoveDone(_gid, _serialNum); - _pending = false; - } -} - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.h b/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.h deleted file mode 100644 index 95aad182b10..00000000000 --- a/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/document/base/globalid.h> -#include <vespa/searchlib/common/serialnum.h> - -namespace proton -{ - -class IGidToLidChangeHandler; - -/* - * Class used to keep track of a pending notifyRemoveDone() call to - * a gid to lid change handler. - */ -class PendingNotifyRemoveDone -{ - IGidToLidChangeHandler *_gidToLidChangeHandler; - document::GlobalId _gid; - search::SerialNum _serialNum; - bool _pending; - -public: - PendingNotifyRemoveDone(); - PendingNotifyRemoveDone(PendingNotifyRemoveDone &&rhs); - PendingNotifyRemoveDone(const PendingNotifyRemoveDone &rhs) = delete; - PendingNotifyRemoveDone &operator=(const PendingNotifyRemoveDone &rhs) = delete; - PendingNotifyRemoveDone &operator=(PendingNotifyRemoveDone &&rhs) = delete; - ~PendingNotifyRemoveDone(); - void setup(IGidToLidChangeHandler &gidToLidChangeHandler, document::GlobalId gid, search::SerialNum serialNum); - void invoke(); -}; - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 93432221e61..57445775df3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -80,7 +80,6 @@ vespa_add_library(searchcore_server STATIC pruneremoveddocumentsjob.cpp putdonecontext.cpp reconfig_params.cpp - remove_batch_done_context.cpp remove_operations_rate_tracker.cpp removedonecontext.cpp removedonetask.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp index 9c9c3fc7eca..32554555984 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.cpp @@ -3,6 +3,7 @@ #include "forcecommitcontext.h" #include "forcecommitdonetask.h" #include <vespa/searchcore/proton/common/docid_limit.h> +#include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> #include <cassert> namespace proton { @@ -10,9 +11,10 @@ namespace proton { ForceCommitContext::ForceCommitContext(vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, PendingLidTrackerBase::Snapshot lidsToCommit, + std::unique_ptr<IPendingGidToLidChanges> pending_gid_to_lid_changes, std::shared_ptr<IDestructorCallback> onDone) : _executor(executor), - _task(std::make_unique<ForceCommitDoneTask>(documentMetaStore)), + _task(std::make_unique<ForceCommitDoneTask>(documentMetaStore, std::move(pending_gid_to_lid_changes))), _committedDocIdLimit(0u), _docIdLimit(nullptr), _lidsToCommit(std::move(lidsToCommit)), diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h index a9987a15da6..73c4ac97f42 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitcontext.h @@ -12,6 +12,7 @@ namespace proton { class ForceCommitDoneTask; struct IDocumentMetaStore; class DocIdLimit; +class IPendingGidToLidChanges; /** * Context class for forced commits that schedules a task when @@ -34,6 +35,7 @@ public: ForceCommitContext(vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, PendingLidTrackerBase::Snapshot lidsToCommit, + std::unique_ptr<IPendingGidToLidChanges> pending_gid_to_lid_changes, std::shared_ptr<IDestructorCallback> onDone); ~ForceCommitContext() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp index 81da2a4ec3e..733c15d55bb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.cpp @@ -2,13 +2,15 @@ #include "forcecommitdonetask.h" #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> +#include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> namespace proton { -ForceCommitDoneTask::ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore) +ForceCommitDoneTask::ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore, std::unique_ptr<IPendingGidToLidChanges> pending_gid_to_lid_changes) : _lidsToReuse(), _holdUnblockShrinkLidSpace(false), - _documentMetaStore(documentMetaStore) + _documentMetaStore(documentMetaStore), + _pending_gid_to_lid_changes(std::move(pending_gid_to_lid_changes)) { } @@ -24,6 +26,9 @@ ForceCommitDoneTask::reuseLids(std::vector<uint32_t> &&lids) void ForceCommitDoneTask::run() { + if (_pending_gid_to_lid_changes) { + _pending_gid_to_lid_changes->notify_done(); + } if (!_lidsToReuse.empty()) { if (_lidsToReuse.size() == 1) { _documentMetaStore.removeComplete(_lidsToReuse[0]); diff --git a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h index cffe4199e84..fe95d1575e9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h +++ b/searchcore/src/vespa/searchcore/proton/server/forcecommitdonetask.h @@ -8,6 +8,7 @@ namespace proton { struct IDocumentMetaStore; +class IPendingGidToLidChanges; /** * Class for task to be executed when a forced commit has completed and @@ -28,9 +29,10 @@ class ForceCommitDoneTask : public vespalib::Executor::Task std::vector<uint32_t> _lidsToReuse; bool _holdUnblockShrinkLidSpace; IDocumentMetaStore &_documentMetaStore; + std::unique_ptr<IPendingGidToLidChanges> _pending_gid_to_lid_changes; public: - ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore); + ForceCommitDoneTask(IDocumentMetaStore &documentMetaStore, std::unique_ptr<IPendingGidToLidChanges> pending_gid_to_lid_changes); ~ForceCommitDoneTask() override; void reuseLids(std::vector<uint32_t> &&lids); @@ -42,7 +44,7 @@ public: void run() override; bool empty() const { - return _lidsToReuse.empty() && !_holdUnblockShrinkLidSpace; + return _lidsToReuse.empty() && !_holdUnblockShrinkLidSpace && !_pending_gid_to_lid_changes; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp index 20ffe203235..23caaf1250b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp @@ -10,18 +10,12 @@ using document::Document; namespace proton { PutDoneContext::PutDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, - IGidToLidChangeHandler &gidToLidChangeHandler, std::shared_ptr<const Document> doc, - const document::GlobalId &gid, uint32_t lid, - search::SerialNum serialNum, bool enableNotifyPut) + uint32_t lid) : OperationDoneContext(std::move(token)), _uncommitted(std::move(uncommitted)), _lid(lid), _docIdLimit(nullptr), - _gidToLidChangeHandler(gidToLidChangeHandler), - _gid(gid), - _serialNum(serialNum), - _enableNotifyPut(enableNotifyPut), _doc(std::move(doc)) { } @@ -31,9 +25,6 @@ PutDoneContext::~PutDoneContext() if (_docIdLimit != nullptr) { _docIdLimit->bumpUpLimit(_lid + 1); } - if (_enableNotifyPut) { - _gidToLidChangeHandler.notifyPutDone(steal(), _gid, _lid, _serialNum); - } } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h index c5e8c558c9e..e7271d8a1b3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h @@ -12,7 +12,6 @@ namespace document { class Document; } namespace proton { class DocIdLimit; -class IGidToLidChangeHandler; /** * Context class for document put operations that acks operation when @@ -26,16 +25,12 @@ class PutDoneContext : public OperationDoneContext IPendingLidTracker::Token _uncommitted; uint32_t _lid; DocIdLimit *_docIdLimit; - IGidToLidChangeHandler &_gidToLidChangeHandler; - document::GlobalId _gid; - search::SerialNum _serialNum; - bool _enableNotifyPut; std::shared_ptr<const document::Document> _doc; public: - PutDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, IGidToLidChangeHandler &gidToLidChangeHandler, + PutDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, std::shared_ptr<const document::Document> doc, - const document::GlobalId &gid, uint32_t lid, search::SerialNum serialNum, bool enableNotifyPut); + uint32_t lid); ~PutDoneContext() override; void registerPutLid(DocIdLimit *docIdLimit) { _docIdLimit = docIdLimit; } diff --git a/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.cpp b/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.cpp deleted file mode 100644 index b0ece5f35a1..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "remove_batch_done_context.h" -#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> - -namespace proton { - -RemoveBatchDoneContext::RemoveBatchDoneContext(vespalib::Executor &executor, - vespalib::Executor::Task::UP task, - IGidToLidChangeHandler &gidToLidChangeHandler, - std::vector<document::GlobalId> gidsToRemove, - search::SerialNum serialNum) - : search::ScheduleTaskCallback(executor, std::move(task)), - _gidToLidChangeHandler(gidToLidChangeHandler), - _gidsToRemove(std::move(gidsToRemove)), - _serialNum(serialNum) -{ -} - -RemoveBatchDoneContext::~RemoveBatchDoneContext() -{ - for (const auto &gid : _gidsToRemove) { - _gidToLidChangeHandler.notifyRemoveDone(gid, _serialNum); - } -} - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.h b/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.h deleted file mode 100644 index 2a93239574a..00000000000 --- a/searchcore/src/vespa/searchcore/proton/server/remove_batch_done_context.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/searchlib/common/scheduletaskcallback.h> -#include <vespa/document/base/globalid.h> -#include <vespa/searchlib/common/serialnum.h> -#include <vector> - -namespace proton -{ - -class IGidToLidChangeHandler; - -/** - * Context class for document batch remove that notifies gid to lid - * change handler about each remove done and schedules a - * task when instance is destroyed. Typically a shared pointer to an - * instance is passed around to multiple worker threads that performs - * portions of a larger task before dropping the shared pointer. - */ -class RemoveBatchDoneContext : public search::ScheduleTaskCallback -{ - IGidToLidChangeHandler &_gidToLidChangeHandler; - std::vector<document::GlobalId> _gidsToRemove; - search::SerialNum _serialNum; - -public: - RemoveBatchDoneContext(vespalib::Executor &executor, - vespalib::Executor::Task::UP task, - IGidToLidChangeHandler &gidToLidChangeHandler, - std::vector<document::GlobalId> gidsToRemove, - search::SerialNum serialNum); - - virtual ~RemoveBatchDoneContext(); -}; - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp index d35e6bb7127..859d8693f6d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp @@ -9,11 +9,10 @@ namespace proton { RemoveDoneContext::RemoveDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, - PendingNotifyRemoveDone &&pendingNotifyRemoveDone, uint32_t lid) + uint32_t lid) : OperationDoneContext(std::move(token)), _executor(executor), _task(), - _pendingNotifyRemoveDone(std::move(pendingNotifyRemoveDone)), _uncommitted(std::move(uncommitted)) { if (lid != 0) { @@ -23,7 +22,6 @@ RemoveDoneContext::RemoveDoneContext(FeedToken token, IPendingLidTracker::Token RemoveDoneContext::~RemoveDoneContext() { - _pendingNotifyRemoveDone.invoke(); ack(); if (_task) { vespalib::Executor::Task::UP res = _executor.execute(std::move(_task)); diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h index 7b6c6be1fe1..485b82dd141 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h @@ -3,7 +3,6 @@ #pragma once #include "operationdonecontext.h" -#include <vespa/searchcore/proton/reference/pending_notify_remove_done.h> #include <vespa/searchcore/proton/common/ipendinglidtracker.h> #include <vespa/vespalib/util/executor.h> #include <vespa/document/base/globalid.h> @@ -26,12 +25,11 @@ class RemoveDoneContext : public OperationDoneContext { vespalib::Executor &_executor; std::unique_ptr<vespalib::Executor::Task> _task; - PendingNotifyRemoveDone _pendingNotifyRemoveDone; IPendingLidTracker::Token _uncommitted; public: RemoveDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, - PendingNotifyRemoveDone &&pendingNotifyRemoveDone, uint32_t lid); + uint32_t lid); ~RemoveDoneContext() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index bf357188766..7c3c796bda3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -5,7 +5,6 @@ #include "ireplayconfig.h" #include "operationdonecontext.h" #include "putdonecontext.h" -#include "remove_batch_done_context.h" #include "removedonecontext.h" #include "updatedonecontext.h" #include <vespa/document/datatype/documenttype.h> @@ -15,7 +14,9 @@ #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> +#include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> #include <vespa/searchlib/common/gatecallback.h> +#include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/vespalib/util/exceptions.h> @@ -48,11 +49,10 @@ private: public: PutDoneContextForMove(FeedToken token, IPendingLidTracker::Token uncommitted, - IGidToLidChangeHandler &gidToLidChangeHandler, std::shared_ptr<const Document> doc, - const document::GlobalId &gid, uint32_t lid, search::SerialNum serialNum, - bool enableNotifyPut, IDestructorCallback::SP moveDoneCtx) - : PutDoneContext(std::move(token), std::move(uncommitted), gidToLidChangeHandler, std::move(doc), gid, lid, serialNum, enableNotifyPut), + uint32_t lid, + IDestructorCallback::SP moveDoneCtx) + : PutDoneContext(std::move(token), std::move(uncommitted),std::move(doc), lid), _moveDoneCtx(std::move(moveDoneCtx)) {} ~PutDoneContextForMove() override = default; @@ -60,31 +60,28 @@ public: std::shared_ptr<PutDoneContext> createPutDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, - IGidToLidChangeHandler &gidToLidChangeHandler, std::shared_ptr<const Document> doc, - const document::GlobalId &gid, uint32_t lid, - SerialNum serialNum, bool enableNotifyPut, + uint32_t lid, IDestructorCallback::SP moveDoneCtx) { std::shared_ptr<PutDoneContext> result; if (moveDoneCtx) { - result = std::make_shared<PutDoneContextForMove>(std::move(token), std::move(uncommitted), gidToLidChangeHandler, - std::move(doc), gid, lid, serialNum, enableNotifyPut, std::move(moveDoneCtx)); + result = std::make_shared<PutDoneContextForMove>(std::move(token), std::move(uncommitted), + std::move(doc), lid, std::move(moveDoneCtx)); } else { - result = std::make_shared<PutDoneContext>(std::move(token), std::move(uncommitted), gidToLidChangeHandler, - std::move(doc), gid, lid, serialNum, enableNotifyPut); + result = std::make_shared<PutDoneContext>(std::move(token), std::move(uncommitted), + std::move(doc), lid); } return result; } std::shared_ptr<PutDoneContext> createPutDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, - IGidToLidChangeHandler &gidToLidChangeHandler, std::shared_ptr<const Document> doc, - const document::GlobalId &gid, uint32_t lid, SerialNum serialNum, bool enableNotifyPut) + uint32_t lid) { - return createPutDoneContext(std::move(token), std::move(uncommitted), gidToLidChangeHandler, std::move(doc), gid, - lid, serialNum, enableNotifyPut, IDestructorCallback::SP()); + return createPutDoneContext(std::move(token), std::move(uncommitted), std::move(doc), + lid, IDestructorCallback::SP()); } std::shared_ptr<UpdateDoneContext> @@ -109,10 +106,10 @@ private: public: RemoveDoneContextForMove(FeedToken token, IPendingLidTracker::Token uncommitted, vespalib::Executor &executor, - IDocumentMetaStore &documentMetaStore, PendingNotifyRemoveDone &&pendingNotifyRemoveDone, + IDocumentMetaStore &documentMetaStore, uint32_t lid, IDestructorCallback::SP moveDoneCtx) : RemoveDoneContext(std::move(token), std::move(uncommitted), executor, - documentMetaStore, std::move(pendingNotifyRemoveDone) ,lid), + documentMetaStore, lid), _moveDoneCtx(std::move(moveDoneCtx)) {} ~RemoveDoneContextForMove() override = default; @@ -120,16 +117,16 @@ public: std::shared_ptr<RemoveDoneContext> createRemoveDoneContext(FeedToken token, IPendingLidTracker::Token uncommitted, vespalib::Executor &executor, - IDocumentMetaStore &documentMetaStore,PendingNotifyRemoveDone &&pendingNotifyRemoveDone, + IDocumentMetaStore &documentMetaStore, uint32_t lid, IDestructorCallback::SP moveDoneCtx) { if (moveDoneCtx) { return std::make_shared<RemoveDoneContextForMove> - (std::move(token), std::move(uncommitted), executor, documentMetaStore, std::move(pendingNotifyRemoveDone), + (std::move(token), std::move(uncommitted), executor, documentMetaStore, lid, std::move(moveDoneCtx)); } else { return std::make_shared<RemoveDoneContext> - (std::move(token), std::move(uncommitted), executor, documentMetaStore, std::move(pendingNotifyRemoveDone), lid); + (std::move(token), std::move(uncommitted), executor, documentMetaStore, lid); } } @@ -243,6 +240,7 @@ StoreOnlyFeedView::forceCommit(SerialNum serialNum, DoneCallback onDone) { internalForceCommit(serialNum, std::make_shared<ForceCommitContext>(_writeService.master(), _metaStore, _pendingLidsForCommit->produceSnapshot(), + _gidToLidChangeHandler.grab_pending_changes(), std::move(onDone))); } @@ -305,17 +303,18 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) putOp.getSubDbId(), putOp.getLid(), putOp.getPrevSubDbId(), putOp.getPrevLid(), _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str()); - PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(putOp, docId.getGlobalId(), docId); + adjustMetaStore(putOp, docId.getGlobalId(), docId); auto uncommitted = get_pending_lid_token(putOp); bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId); if (putOp.getValidDbdId(_params._subDbId)) { - const document::GlobalId &gid = docId.getGlobalId(); + if (putOp.changedDbdId() && useDocumentMetaStore(serialNum)) { + _gidToLidChangeHandler.notifyPut(token, docId.getGlobalId(), putOp.getLid(), serialNum); + } std::shared_ptr<PutDoneContext> onWriteDone = createPutDoneContext(std::move(token), std::move(uncommitted), - _gidToLidChangeHandler, doc, gid, putOp.getLid(), serialNum, - putOp.changedDbdId() && useDocumentMetaStore(serialNum)); + doc, putOp.getLid()); putSummary(serialNum, putOp.getLid(), doc, onWriteDone); putAttributes(serialNum, putOp.getLid(), *doc, onWriteDone); putIndexedFields(serialNum, putOp.getLid(), doc, onWriteDone); @@ -323,7 +322,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp) if (docAlreadyExists && putOp.changedDbdId()) { assert(!putOp.getValidDbdId(_params._subDbId)); internalRemove(std::move(token), _pendingLidsForCommit->produce(putOp.getPrevLid()), serialNum, - std::move(pendingNotifyRemoveDone), putOp.getPrevLid(), IDestructorCallback::SP()); + putOp.getPrevLid(), IDestructorCallback::SP()); } } @@ -574,7 +573,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithDocI _params._docTypeName.toString().c_str(), serialNum, docId.toString().c_str(), rmOp.getSubDbId(), rmOp.getLid(), rmOp.getPrevSubDbId(), rmOp.getPrevLid(), _params._subDbId); - PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, docId.getGlobalId(), docId); + adjustMetaStore(rmOp, docId.getGlobalId(), docId); auto uncommitted = get_pending_lid_token(rmOp); if (rmOp.getValidDbdId(_params._subDbId)) { @@ -587,7 +586,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithDocI if (rmOp.changedDbdId()) { assert(!rmOp.getValidDbdId(_params._subDbId)); internalRemove(std::move(token), _pendingLidsForCommit->produce(rmOp.getPrevLid()), serialNum, - std::move(pendingNotifyRemoveDone), rmOp.getPrevLid(), IDestructorCallback::SP()); + rmOp.getPrevLid(), IDestructorCallback::SP()); } } } @@ -599,13 +598,13 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithGid assert(rmOp.notMovingLidInSameSubDb()); const SerialNum serialNum = rmOp.getSerialNum(); DocumentId dummy; - PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, rmOp.getGlobalId(), dummy); + adjustMetaStore(rmOp, rmOp.getGlobalId(), dummy); auto uncommitted = _pendingLidsForCommit->produce(rmOp.getLid()); if (rmOp.getValidPrevDbdId(_params._subDbId)) { if (rmOp.changedDbdId()) { assert(!rmOp.getValidDbdId(_params._subDbId)); - internalRemove(std::move(token), _pendingLidsForCommit->produce(rmOp.getPrevLid()), serialNum, std::move(pendingNotifyRemoveDone), + internalRemove(std::move(token), _pendingLidsForCommit->produce(rmOp.getPrevLid()), serialNum, rmOp.getPrevLid(), IDestructorCallback::SP()); } } @@ -613,23 +612,22 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithGid void StoreOnlyFeedView::internalRemove(FeedToken token, IPendingLidTracker::Token uncommitted, SerialNum serialNum, - PendingNotifyRemoveDone &&pendingNotifyRemoveDone, Lid lid, + Lid lid, IDestructorCallback::SP moveDoneCtx) { bool explicitReuseLid = _lidReuseDelayer.delayReuse(lid); std::shared_ptr<RemoveDoneContext> onWriteDone; onWriteDone = createRemoveDoneContext(std::move(token), std::move(uncommitted),_writeService.master(), _metaStore, - std::move(pendingNotifyRemoveDone), (explicitReuseLid ? lid : 0u), + (explicitReuseLid ? lid : 0u), std::move(moveDoneCtx)); removeSummary(serialNum, lid, onWriteDone); removeAttributes(serialNum, lid, onWriteDone); removeIndexedFields(serialNum, lid, onWriteDone); } -PendingNotifyRemoveDone +void StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const GlobalId & gid, const DocumentId &docId) { - PendingNotifyRemoveDone pendingNotifyRemoveDone; const SerialNum serialNum = op.getSerialNum(); if (useDocumentMetaStore(serialNum)) { if (op.getValidDbdId(_params._subDbId)) { @@ -644,13 +642,11 @@ StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const GlobalId & } else if (op.getValidPrevDbdId(_params._subDbId)) { vespalib::Gate gate; _gidToLidChangeHandler.notifyRemove(std::make_shared<search::GateCallback>(gate), gid, serialNum); - pendingNotifyRemoveDone.setup(_gidToLidChangeHandler, gid, serialNum); gate.await(); removeMetaData(_metaStore, gid, docId, op, _params._subDbType == SubDbType::REMOVED); } _metaStore.commit(serialNum, serialNum); } - return pendingNotifyRemoveDone; } void @@ -695,8 +691,7 @@ StoreOnlyFeedView::removeDocuments(const RemoveDocumentsOperation &op, bool remo } else { removeBatchDoneTask = makeLambdaTask([]() {}); } - onWriteDone = std::make_shared<RemoveBatchDoneContext>(_writeService.master(), std::move(removeBatchDoneTask), - _gidToLidChangeHandler, std::move(gidsToRemove), serialNum); + onWriteDone = std::make_shared<search::ScheduleTaskCallback>(_writeService.master(), std::move(removeBatchDoneTask)); if (remove_index_and_attributes) { removeIndexedFields(serialNum, lidsToRemove, onWriteDone); removeAttributes(serialNum, lidsToRemove, onWriteDone); @@ -767,20 +762,21 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: moveOp.getSubDbId(), moveOp.getLid(), moveOp.getPrevSubDbId(), moveOp.getPrevLid(), _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str()); - PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId.getGlobalId(), docId); + adjustMetaStore(moveOp, docId.getGlobalId(), docId); bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId); if (moveOp.getValidDbdId(_params._subDbId)) { - const document::GlobalId &gid = docId.getGlobalId(); + if (moveOp.changedDbdId() && useDocumentMetaStore(serialNum)) { + _gidToLidChangeHandler.notifyPut(FeedToken(), docId.getGlobalId(), moveOp.getLid(), serialNum); + } std::shared_ptr<PutDoneContext> onWriteDone = createPutDoneContext(FeedToken(), _pendingLidsForCommit->produce(moveOp.getLid()), - _gidToLidChangeHandler, doc, gid, moveOp.getLid(), serialNum, - moveOp.changedDbdId() && useDocumentMetaStore(serialNum), doneCtx); + doc, moveOp.getLid(), doneCtx); putSummary(serialNum, moveOp.getLid(), doc, onWriteDone); putAttributes(serialNum, moveOp.getLid(), *doc, onWriteDone); putIndexedFields(serialNum, moveOp.getLid(), doc, onWriteDone); } if (docAlreadyExists && moveOp.changedDbdId()) { - internalRemove(FeedToken(), _pendingLidsForCommit->produce(moveOp.getPrevLid()), serialNum, std::move(pendingNotifyRemoveDone), moveOp.getPrevLid(), doneCtx); + internalRemove(FeedToken(), _pendingLidsForCommit->produce(moveOp.getPrevLid()), serialNum, moveOp.getPrevLid(), doneCtx); } } @@ -820,6 +816,7 @@ StoreOnlyFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op) getDocumentMetaStore()->get().compactLidSpace(op.getLidLimit()); auto commitContext(std::make_shared<ForceCommitContext>(_writeService.master(), _metaStore, _pendingLidsForCommit->produceSnapshot(), + _gidToLidChangeHandler.grab_pending_changes(), DoneCallback())); commitContext->holdUnblockShrinkLidSpace(); internalForceCommit(serialNum, commitContext); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index da7d5e53a88..9927c93add4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -16,7 +16,6 @@ #include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h> #include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h> #include <vespa/searchcore/proton/persistenceengine/resulthandler.h> -#include <vespa/searchcore/proton/reference/pending_notify_remove_done.h> #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/searchlib/query/base.h> #include <vespa/vespalib/util/threadstackexecutorbase.h> @@ -169,7 +168,7 @@ private: return replaySerialNum > _params._flushedDocumentMetaStoreSerialNum; } - PendingNotifyRemoveDone adjustMetaStore(const DocumentOperation &op, const document::GlobalId & gid, const document::DocumentId &docId); + void adjustMetaStore(const DocumentOperation &op, const document::GlobalId & gid, const document::DocumentId &docId); void internalPut(FeedToken token, const PutOperation &putOp); void internalUpdate(FeedToken token, const UpdateOperation &updOp); @@ -182,7 +181,6 @@ private: size_t removeDocuments(const RemoveDocumentsOperation &op, bool remove_index_and_attribute_fields); void internalRemove(FeedToken token, IPendingLidTracker::Token uncommitted, SerialNum serialNum, - PendingNotifyRemoveDone &&pendingNotifyRemoveDone, Lid lid, std::shared_ptr<search::IDestructorCallback> moveDoneCtx); IPendingLidTracker::Token get_pending_lid_token(const DocumentOperation &op); diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h index f9531486e9b..976cdc70048 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h @@ -3,6 +3,7 @@ #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h> +#include <vespa/searchcore/proton/reference/i_pending_gid_to_lid_changes.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -42,9 +43,9 @@ public: _removes.emplace_back(docTypeName, keepNames); } - void notifyPutDone(IDestructorCallbackSP, document::GlobalId, uint32_t, SerialNum) override { } + void notifyPut(IDestructorCallbackSP, document::GlobalId, uint32_t, SerialNum) override { } void notifyRemove(IDestructorCallbackSP, document::GlobalId, SerialNum) override { } - void notifyRemoveDone(document::GlobalId, SerialNum) override { } + std::unique_ptr<IPendingGidToLidChanges> grab_pending_changes() override { return {}; } void assertAdds(const std::vector<AddEntry> &expAdds) { diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index da77e29dbb0..daa85c91b2c 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -49,6 +49,7 @@ using search::tensor::NearestNeighborIndexSaver; using search::tensor::PrepareResult; using search::tensor::TensorAttribute; using vespalib::eval::TensorSpec; +using vespalib::eval::CellType; using vespalib::eval::ValueType; using vespalib::eval::Value; using vespalib::eval::EngineOrFactory; @@ -228,11 +229,11 @@ class MockNearestNeighborIndexFactory : public NearestNeighborIndexFactory { std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, size_t vector_size, - ValueType::CellType cell_type, + CellType cell_type, const search::attribute::HnswIndexParams& params) const override { (void) vector_size; (void) params; - assert(cell_type == ValueType::CellType::DOUBLE); + assert(cell_type == CellType::DOUBLE); return std::make_unique<MockNearestNeighborIndex>(vectors); } }; diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index f6ca0bd1427..23cb3831b6d 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -25,7 +25,7 @@ using search::AttributeVector; using search::BitVector; using vespalib::eval::Value; using vespalib::eval::ValueType; -using CellType = vespalib::eval::ValueType::CellType; +using vespalib::eval::CellType; using vespalib::eval::TensorSpec; using vespalib::eval::EngineOrFactory; using search::tensor::DistanceFunction; diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index a9e24e056f2..06fb95089fd 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -33,7 +33,7 @@ void verify_geo_miles(const DistanceFunction *dist_fun, TEST(DistanceFunctionsTest, euclidean_gives_expected_score) { - auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + auto ct = vespalib::eval::CellType::DOUBLE; auto euclid = make_distance_function(DistanceMetric::Euclidean, ct); @@ -54,7 +54,7 @@ TEST(DistanceFunctionsTest, euclidean_gives_expected_score) TEST(DistanceFunctionsTest, angular_gives_expected_score) { - auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + auto ct = vespalib::eval::CellType::DOUBLE; auto angular = make_distance_function(DistanceMetric::Angular, ct); @@ -109,7 +109,7 @@ TEST(DistanceFunctionsTest, angular_gives_expected_score) TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) { - auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + auto ct = vespalib::eval::CellType::DOUBLE; auto innerproduct = make_distance_function(DistanceMetric::InnerProduct, ct); @@ -144,7 +144,7 @@ TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) TEST(DistanceFunctionsTest, hamming_gives_expected_score) { - auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + auto ct = vespalib::eval::CellType::DOUBLE; auto hamming = make_distance_function(DistanceMetric::Hamming, ct); @@ -184,7 +184,7 @@ TEST(DistanceFunctionsTest, hamming_gives_expected_score) TEST(GeoDegreesTest, gives_expected_score) { - auto ct = vespalib::eval::ValueType::CellType::DOUBLE; + auto ct = vespalib::eval::CellType::DOUBLE; auto geodeg = make_distance_function(DistanceMetric::GeoDegrees, ct); std::vector<double> g1_sfo{37.61, -122.38}; diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 2bcdbc8ecbf..2168bbe4276 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -310,9 +310,7 @@ AttributeVector::createAttributeHeader(vespalib::stringref fileName) const { return attribute::AttributeHeader(fileName, getConfig().basicType(), getConfig().collectionType(), - (getConfig().basicType().type() == BasicType::Type::TENSOR - ? getConfig().tensorType() - : vespalib::eval::ValueType::error_type()), + getConfig().tensorType(), getEnumeratedSave(), getConfig().predicateParams(), getConfig().hnsw_index_params(), diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp index dd685ce5c43..85b7e8f89e8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp @@ -7,8 +7,7 @@ using search::tensor::DenseTensorAttribute; using vespalib::ConstArrayRef; using vespalib::tensor::MutableDenseTensorView; using vespalib::eval::TypedCells; - -using CellType = vespalib::eval::ValueType::CellType; +using vespalib::eval::CellType; namespace search::queryeval { diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp index 0bb6f339455..aca14a1575e 100644 --- a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp @@ -28,7 +28,7 @@ make_random_level_generator(uint32_t m) std::unique_ptr<NearestNeighborIndex> DefaultNearestNeighborIndexFactory::make(const DocVectorAccess& vectors, size_t vector_size, - vespalib::eval::ValueType::CellType cell_type, + vespalib::eval::CellType cell_type, const search::attribute::HnswIndexParams& params) const { (void) vector_size; diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h index 6a9ded92b60..67a19a5431a 100644 --- a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h +++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h @@ -13,7 +13,7 @@ class DefaultNearestNeighborIndexFactory : public NearestNeighborIndexFactory { public: std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, size_t vector_size, - vespalib::eval::ValueType::CellType cell_type, + vespalib::eval::CellType cell_type, const search::attribute::HnswIndexParams& params) const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index 1abc3800d97..ddbb956838b 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -9,7 +9,7 @@ using vespalib::datastore::Handle; using vespalib::tensor::MutableDenseTensorView; using vespalib::eval::Value; using vespalib::eval::ValueType; -using CellType = vespalib::eval::ValueType::CellType; +using CellType = vespalib::eval::CellType; namespace search::tensor { diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp index a868dfe191b..81b27b56258 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp @@ -4,44 +4,45 @@ #include "distance_functions.h" using search::attribute::DistanceMetric; +using vespalib::eval::CellType; using vespalib::eval::ValueType; namespace search::tensor { DistanceFunction::UP -make_distance_function(DistanceMetric variant, ValueType::CellType cell_type) +make_distance_function(DistanceMetric variant, CellType cell_type) { switch (variant) { case DistanceMetric::Euclidean: - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return std::make_unique<SquaredEuclideanDistance<float>>(); } else { return std::make_unique<SquaredEuclideanDistance<double>>(); } break; case DistanceMetric::Angular: - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return std::make_unique<AngularDistance<float>>(); } else { return std::make_unique<AngularDistance<double>>(); } break; case DistanceMetric::GeoDegrees: - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return std::make_unique<GeoDegreesDistance<float>>(); } else { return std::make_unique<GeoDegreesDistance<double>>(); } break; case DistanceMetric::InnerProduct: - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return std::make_unique<InnerProductDistance<float>>(); } else { return std::make_unique<InnerProductDistance<double>>(); } break; case DistanceMetric::Hamming: - if (cell_type == ValueType::CellType::FLOAT) { + if (cell_type == CellType::FLOAT) { return std::make_unique<HammingDistance<float>>(); } else { return std::make_unique<HammingDistance<double>>(); diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h index c86e40279bc..abb1f503694 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h @@ -14,6 +14,6 @@ namespace search::tensor { **/ DistanceFunction::UP make_distance_function(search::attribute::DistanceMetric variant, - vespalib::eval::ValueType::CellType cell_type); + vespalib::eval::CellType cell_type); } diff --git a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h index f5481a680a3..c962e919d95 100644 --- a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h @@ -26,7 +26,7 @@ public: virtual bool supports_extract_dense_view() const = 0; virtual bool supports_get_tensor_ref() const = 0; - virtual vespalib::eval::ValueType getTensorType() const = 0; + virtual const vespalib::eval::ValueType & getTensorType() const = 0; /** * Gets custom state for this tensor attribute by inserting it into the given Slime inserter. diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp index 1e376faa4d3..6a0dbfb9f48 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp @@ -60,7 +60,7 @@ ImportedTensorAttributeVectorReadGuard::get_tensor_ref(uint32_t docid) const return _target_tensor_attribute.get_tensor_ref(getTargetLid(docid)); } -vespalib::eval::ValueType +const vespalib::eval::ValueType & ImportedTensorAttributeVectorReadGuard::getTensorType() const { return _target_tensor_attribute.getTensorType(); diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h index 3abac4e532e..a3ffc27b153 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h @@ -36,7 +36,7 @@ public: const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; bool supports_extract_dense_view() const override { return _target_tensor_attribute.supports_extract_dense_view(); } bool supports_get_tensor_ref() const override { return _target_tensor_attribute.supports_get_tensor_ref(); } - vespalib::eval::ValueType getTensorType() const override; + const vespalib::eval::ValueType &getTensorType() const override; void get_state(const vespalib::slime::Inserter& inserter) const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h index 089119944a7..e5c15266ceb 100644 --- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h +++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h @@ -20,7 +20,7 @@ public: virtual ~NearestNeighborIndexFactory() {} virtual std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors, size_t vector_size, - vespalib::eval::ValueType::CellType cell_type, + vespalib::eval::CellType cell_type, const search::attribute::HnswIndexParams& params) const = 0; }; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 39e35af3174..0748329694c 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -202,7 +202,7 @@ TensorAttribute::get_tensor_ref(uint32_t docid) const abort(); // Needed to avoid compile error } -vespalib::eval::ValueType +const vespalib::eval::ValueType & TensorAttribute::getTensorType() const { return getConfig().tensorType(); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index 582fad59828..b88ffcf0f2c 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -51,7 +51,7 @@ public: const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; bool supports_extract_dense_view() const override { return false; } bool supports_get_tensor_ref() const override { return false; } - vespalib::eval::ValueType getTensorType() const override; + const vespalib::eval::ValueType & getTensorType() const override; void get_state(const vespalib::slime::Inserter& inserter) const override; void clearDocs(DocId lidLow, DocId lidLimit) override; void onShrinkLidSpace() override; diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java index a2b82978a26..e0e3469e257 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java @@ -29,6 +29,7 @@ public class MetricReceiver { private volatile Map<String, MetricSettings> metricSettings; private static final class NullCounter extends Counter { + NullCounter() { super(null, null, null); } @@ -72,18 +73,23 @@ public class MetricReceiver { public PointBuilder builder() { return super.builder(); } + } public static final class MockReceiver extends MetricReceiver { + private final ThreadLocalDirectory<Bucket, Sample> collection; + private MockReceiver(ThreadLocalDirectory<Bucket, Sample> collection) { super(collection, null); this.collection = collection; } + public MockReceiver() { this(new ThreadLocalDirectory<>(new MetricUpdater())); } - /** gathers all data since last snapshot */ + + /** Gathers all data since last snapshot */ public Bucket getSnapshot() { final Bucket merged = new Bucket(); for (Bucket b : collection.fetch()) { @@ -91,13 +97,16 @@ public class MetricReceiver { } return merged; } - /** utility method for testing */ + + /** Utility method for testing */ public Point point(String dim, String val) { return pointBuilder().set(dim, val).build(); } + } private static final class NullReceiver extends MetricReceiver { + NullReceiver() { super(null, null); } @@ -164,21 +173,18 @@ public class MetricReceiver { * {@link #declareGauge(String)}, or {@link #declareGauge(String, Point)} * instead. * - * @param s - * a single simple containing all meta data necessary to update a - * metric + * @param sample a single simple containing all meta data necessary to update a metric */ - public void update(Sample s) { + public void update(Sample sample) { // pass around the receiver instead of histogram settings to avoid reading any volatile if unnecessary - s.setReceiver(this); - metricsCollection.update(s); + sample.setReceiver(this); + metricsCollection.update(sample); } /** * Declare a counter metric without setting any default position. * - * @param name - * the name of the metric + * @param name the name of the metric * @return a thread-safe counter */ public Counter declareCounter(String name) { @@ -189,11 +195,8 @@ public class MetricReceiver { * Declare a counter metric, with default dimension values as given. Create * the point argument by using a builder from {@link #pointBuilder()}. * - * @param name - * the name of the metric - * @param boundDimensions - * dimensions which have a fixed value in the life cycle of the - * metric object or null + * @param name the name of the metric + * @param boundDimensions dimensions which have a fixed value in the life cycle of the metric object or null * @return a thread-safe counter with given default values */ public Counter declareCounter(String name, Point boundDimensions) { @@ -203,8 +206,7 @@ public class MetricReceiver { /** * Declare a gauge metric with any default position. * - * @param name - * the name of the metric + * @param name the name of the metric * @return a thread-safe gauge instance */ public Gauge declareGauge(String name) { @@ -215,21 +217,12 @@ public class MetricReceiver { * Declare a gauge metric, with default dimension values as given. Create * the point argument by using a builder from {@link #pointBuilder()}. * - * @param name - * the name of the metric - * @param boundDimensions - * dimensions which have a fixed value in the life cycle of the - * metric object or null + * @param name the name of the metric + * @param boundDimensions dimensions which have a fixed value in the life cycle of the metric object or null * @return a thread-safe gauge metric */ public Gauge declareGauge(String name, Point boundDimensions) { - Optional<Point> optionalOfBoundDimensions; - if (boundDimensions == null) { - optionalOfBoundDimensions = Optional.empty(); - } else { - optionalOfBoundDimensions = Optional.of(boundDimensions); - } - return declareGauge(name, optionalOfBoundDimensions, null); + return declareGauge(name, Optional.ofNullable(boundDimensions), null); } /** @@ -238,13 +231,9 @@ public class MetricReceiver { * MetricSettings instances are built using * {@link MetricSettings.Builder}. * - * @param name - * the name of the metric - * @param boundDimensions - * an optional of dimensions which have a fixed value in the life - * cycle of the metric object - * @param customSettings - * any optional settings + * @param name the name of the metric + * @param boundDimensions an optional of dimensions which have a fixed value in the life cycle of the metric object + * @param customSettings any optional settings * @return a thread-safe gauge metric */ public Gauge declareGauge(String name, Optional<Point> boundDimensions, MetricSettings customSettings) { @@ -283,14 +272,8 @@ public class MetricReceiver { /** * Add how to build a histogram for a given metric. * - * <p> - * Do note, this is not part of the public API. - * </p> - * - * @param metricName - * the metric where samples should be put in a histogram - * @param definition - * settings for a histogram + * @param metricName the metric where samples should be put in a histogram + * @param definition settings for a histogram */ void addMetricDefinition(String metricName, MetricSettings definition) { synchronized (histogramDefinitionsLock) { @@ -304,15 +287,9 @@ public class MetricReceiver { } /** - * Get how to build a histogram for a given metric, or null if no histogram - * should be created. - * - * <p> - * Do note, this is not part of the public API. - * </p> + * Get how to build a histogram for a given metric, or null if no histogram should be created. * - * @param metricName - * the name of an arbitrary metric + * @param metricName the name of an arbitrary metric * @return the corresponding histogram definition or null */ MetricSettings getMetricDefinition(String metricName) { diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index adb50465adc..e075f57514b 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -61,7 +61,7 @@ skip_thread bool default=false use_direct_storageapi_rpc bool default=false ## The number of network (FNET) threads used by the shared rpc resource. -rpc.num_network_threads int default=1 +rpc.num_network_threads int default=2 ## The number of (FNET) RPC targets to use per node in the cluster. ## diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java index 8cfbfd204ba..7119bde7a09 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java @@ -4,6 +4,7 @@ package ai.vespa.hosted.plugin; import ai.vespa.hosted.api.ControllerHttpClient; import ai.vespa.hosted.api.Properties; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.yolean.Exceptions; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -15,6 +16,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.joining; /** * Base class for hosted Vespa plugin mojos. @@ -61,7 +66,11 @@ public abstract class AbstractVespaMojo extends AbstractMojo { throw e; } catch (Exception e) { - throw new MojoExecutionException("Execution failed for application " + name(), e); + String message = "Execution failed for application " + name() + ":\n" + Exceptions.toMessageString(e); + if (e.getSuppressed().length > 0) + message += "\nSuppressed:\n" + Stream.of(e.getSuppressed()).map(Exceptions::toMessageString).collect(joining("\n")); + + throw new MojoExecutionException(message, e); } } diff --git a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java index de1040852f5..4a24cdcc7bf 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java @@ -10,6 +10,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Random; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Predicate; @@ -72,6 +73,9 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte /** Returns the items in this as an immutable list. */ public final List<Type> asList() { return items; } + /** Returns the items in this as a set. */ + public final Set<Type> asSet() { return new HashSet<>(items); } + /** Returns the items in this as an immutable list after mapping with the given function. */ public final <OtherType> List<OtherType> mapToList(Function<Type, OtherType> mapper) { return items.stream().map(mapper).collect(toUnmodifiableList()); diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java index d5235caef9f..5ab1c88775a 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java @@ -62,14 +62,13 @@ import java.util.List; * example. * </p> * - * @param AGGREGATOR - * the type input data is aggregated into - * @param SAMPLE - * the type of input data + * @param <AGGREGATOR> the type input data is aggregated into + * @param <SAMPLE> the type of input data * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { + /** * Factory interface to create the data container for each generation of * samples, and putting data into it. @@ -85,12 +84,11 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * need to implement both. * </p> * - * @param AGGREGATOR - * The type of the data container to produce - * @param SAMPLE - * The type of the incoming data to store in the container. + * @param <AGGREGATOR> the type of the data container to produce + * @param <SAMPLE> the type of the incoming data to store in the container. */ public interface Updater<AGGREGATOR, SAMPLE> { + /** * Create data container to receive produced data. This is invoked once * on every instance every time ThreadLocalDirectory.fetch() is invoked. @@ -137,7 +135,7 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * * @return a fresh structure to receive data */ - public AGGREGATOR createGenerationInstance(AGGREGATOR previous); + AGGREGATOR createGenerationInstance(AGGREGATOR previous); /** * Insert a data element of type S into the current generation of data @@ -180,7 +178,8 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * the data to insert * @return the new current value, may be the same as previous */ - public AGGREGATOR update(AGGREGATOR current, SAMPLE x); + AGGREGATOR update(AGGREGATOR current, SAMPLE x); + } /** @@ -188,14 +187,12 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * ThreadLocalDirectory without resetting the local instances in each * thread. * - * @param <AGGREGATOR> - * as for {@link Updater} - * @param <SAMPLE> - * as for {@link Updater} + * @param <AGGREGATOR> as for {@link Updater} + * @param <SAMPLE> as for {@link Updater} * @see ThreadLocalDirectory#view() */ - public interface ObservableUpdater<AGGREGATOR, SAMPLE> extends - Updater<AGGREGATOR, SAMPLE> { + public interface ObservableUpdater<AGGREGATOR, SAMPLE> extends Updater<AGGREGATOR, SAMPLE> { + /** * Create an application specific copy of the AGGREGATOR for a thread. * @@ -203,7 +200,8 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * the AGGREGATOR instance to copy * @return a copy of the incoming parameter */ - public AGGREGATOR copy(AGGREGATOR current); + AGGREGATOR copy(AGGREGATOR current); + } private final ThreadLocal<LocalInstance<AGGREGATOR, SAMPLE>> local = new ThreadLocal<>(); @@ -268,8 +266,7 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * to have been instantiated with an updater implementing ObservableUpdater. * * @return a list of a copy of the current data in all producer threads - * @throws IllegalStateException - * if the updater does not implement {@link ObservableUpdater} + * @throws IllegalStateException if the updater does not implement {@link ObservableUpdater} */ public List<AGGREGATOR> view() { if (observableUpdater == null) { @@ -310,8 +307,7 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { /** * Input data from a producer thread. * - * @param x - * the data to insert + * @param x the data to insert */ public void update(SAMPLE x) { update(x, getOrCreateLocal()); @@ -330,10 +326,8 @@ public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { * calls necessary to update(SAMPLE, LocalInstance<AGGREGATOR, SAMPLE>). * </p> * - * @param x - * the data to insert - * @param localInstance - * the local data insertion instance + * @param x the data to insert + * @param localInstance the local data insertion instance */ public void update(SAMPLE x, LocalInstance<AGGREGATOR, SAMPLE> localInstance) { boolean isRegistered; diff --git a/vespalib/src/tests/spin_lock/spin_lock_test.cpp b/vespalib/src/tests/spin_lock/spin_lock_test.cpp index 5ba0ca16222..3542bd5d51f 100644 --- a/vespalib/src/tests/spin_lock/spin_lock_test.cpp +++ b/vespalib/src/tests/spin_lock/spin_lock_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <array> using namespace vespalib; diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json index e026559b283..4aa8775940e 100644 --- a/zkfacade/abi-spec.json +++ b/zkfacade/abi-spec.json @@ -68,6 +68,7 @@ "methods": [ "public static com.yahoo.vespa.curator.Curator create(java.lang.String)", "public static com.yahoo.vespa.curator.Curator create(java.lang.String, java.util.Optional)", + "public void <init>(com.yahoo.cloud.config.CuratorConfig, com.yahoo.vespa.zookeeper.VespaZooKeeperServer)", "public void <init>(com.yahoo.cloud.config.ConfigserverConfig, com.yahoo.vespa.zookeeper.VespaZooKeeperServer)", "protected void <init>(java.lang.String, java.lang.String, java.util.function.Function)", "public java.lang.String connectionSpec()", diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/ConnectionSpec.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/ConnectionSpec.java new file mode 100644 index 00000000000..4409291419a --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/ConnectionSpec.java @@ -0,0 +1,102 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator; + +import com.yahoo.net.HostName; + +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +/** + * A connection spec for Curator. + * + * @author mpolden + */ +class ConnectionSpec { + + private final String local; + private final String ensemble; + private final int ensembleSize; + + private ConnectionSpec(String local, String ensemble, int ensembleSize) { + this.local = requireNonEmpty(local, "local spec"); + this.ensemble = requireNonEmpty(ensemble, "ensemble spec"); + this.ensembleSize = ensembleSize; + } + + /** Returns the local spec. This may be a subset of the ensemble spec */ + public String local() { + return local; + } + + /** Returns the ensemble spec. This always contains all nodes in the ensemble */ + public String ensemble() { + return ensemble; + } + + /** Returns the number of servers in the ensemble */ + public int ensembleSize() { + return ensembleSize; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConnectionSpec that = (ConnectionSpec) o; + return ensembleSize == that.ensembleSize && + local.equals(that.local) && + ensemble.equals(that.ensemble); + } + + @Override + public int hashCode() { + return Objects.hash(local, ensemble, ensembleSize); + } + + public static ConnectionSpec create(String spec) { + return create(spec, spec); + } + + public static ConnectionSpec create(String localSpec, String ensembleSpec) { + return new ConnectionSpec(localSpec, ensembleSpec, ensembleSpec.split(",").length); + } + + public static <T> ConnectionSpec create(List<T> servers, + Function<T, String> hostnameGetter, + Function<T, Integer> portGetter, + boolean localhostAffinity) { + String localSpec = createSpec(servers, hostnameGetter, portGetter, localhostAffinity); + String ensembleSpec = localhostAffinity ? createSpec(servers, hostnameGetter, portGetter, false) : localSpec; + return new ConnectionSpec(localSpec, ensembleSpec, servers.size()); + } + + private static <T> String createSpec(List<T> servers, + Function<T, String> hostnameGetter, + Function<T, Integer> portGetter, + boolean localhostAffinity) { + String thisServer = HostName.getLocalhost(); + StringBuilder connectionSpec = new StringBuilder(); + for (var server : servers) { + if (localhostAffinity && !thisServer.equals(hostnameGetter.apply(server))) continue; + connectionSpec.append(hostnameGetter.apply(server)); + connectionSpec.append(':'); + connectionSpec.append(portGetter.apply(server)); + connectionSpec.append(','); + } + if (localhostAffinity && connectionSpec.length() == 0) { + throw new IllegalArgumentException("Unable to create connect string to localhost: " + + "There is no localhost server specified in config"); + } + if (connectionSpec.length() > 0) { + connectionSpec.setLength(connectionSpec.length() - 1); // Remove trailing comma + } + return connectionSpec.toString(); + } + + private static String requireNonEmpty(String s, String field) { + if (Objects.requireNonNull(s).isEmpty()) throw new IllegalArgumentException(field + " must be non-empty"); + return s; + } + +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 6cbfa274c56..5966ef77877 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.curator; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.io.IOUtils; -import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.recipes.CuratorCounter; @@ -31,6 +31,7 @@ import java.io.File; import java.time.Duration; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -55,81 +56,67 @@ public class Curator implements AutoCloseable { private static final Duration ZK_CONNECTION_TIMEOUT = Duration.ofSeconds(30); private static final Duration BASE_SLEEP_TIME = Duration.ofSeconds(1); private static final int MAX_RETRIES = 10; + private static final RetryPolicy DEFAULT_RETRY_POLICY = new ExponentialBackoffRetry((int) BASE_SLEEP_TIME.toMillis(), MAX_RETRIES); - protected final RetryPolicy retryPolicy; + protected final RetryPolicy retryPolicy = DEFAULT_RETRY_POLICY; private final CuratorFramework curatorFramework; - private final String connectionSpec; // May be a subset of the servers in the ensemble - private final String zooKeeperEnsembleConnectionSpec; - private final int zooKeeperEnsembleCount; + private final ConnectionSpec connectionSpec; // All lock keys, to allow re-entrancy. This will grow forever, but this should be too slow to be a problem private final ConcurrentHashMap<Path, Lock> locks = new ConcurrentHashMap<>(); /** Creates a curator instance from a comma-separated string of ZooKeeper host:port strings */ public static Curator create(String connectionSpec) { - return new Curator(connectionSpec, connectionSpec, Optional.of(ZK_CLIENT_CONFIG_FILE)); + return new Curator(ConnectionSpec.create(connectionSpec), Optional.of(ZK_CLIENT_CONFIG_FILE)); } // For testing only, use Optional.empty for clientConfigFile parameter to create default zookeeper client config public static Curator create(String connectionSpec, Optional<File> clientConfigFile) { - return new Curator(connectionSpec, connectionSpec, clientConfigFile); + return new Curator(ConnectionSpec.create(connectionSpec), clientConfigFile); } - // Depend on ZooKeeperServer to make sure it is started first - // TODO: Move zookeeperserver config out of configserverconfig (requires update of controller services.xml as well) @Inject - public Curator(ConfigserverConfig configserverConfig, VespaZooKeeperServer server) { - this(configserverConfig, Optional.of(ZK_CLIENT_CONFIG_FILE)); + public Curator(CuratorConfig curatorConfig, @SuppressWarnings("unused") VespaZooKeeperServer server) { + // Depends on ZooKeeperServer to make sure it is started first + this(ConnectionSpec.create(curatorConfig.server(), + CuratorConfig.Server::hostname, + CuratorConfig.Server::port, + curatorConfig.zookeeperLocalhostAffinity()), + Optional.of(ZK_CLIENT_CONFIG_FILE)); } - Curator(ConfigserverConfig configserverConfig, Optional<File> clientConfigFile) { - this(createConnectionSpec(configserverConfig), createEnsembleConnectionSpec(configserverConfig), clientConfigFile); + // TODO: This can be removed when this package is no longer public API. + public Curator(ConfigserverConfig configserverConfig, @SuppressWarnings("unused") VespaZooKeeperServer server) { + this(ConnectionSpec.create(configserverConfig.zookeeperserver(), + ConfigserverConfig.Zookeeperserver::hostname, + ConfigserverConfig.Zookeeperserver::port, + configserverConfig.zookeeperLocalhostAffinity()), + Optional.of(ZK_CLIENT_CONFIG_FILE)); } - private Curator(String connectionSpec, String zooKeeperEnsembleConnectionSpec, Optional<File> clientConfigFile) { - this(connectionSpec, - zooKeeperEnsembleConnectionSpec, - (retryPolicy) -> CuratorFrameworkFactory - .builder() - .retryPolicy(retryPolicy) - .sessionTimeoutMs((int) ZK_SESSION_TIMEOUT.toMillis()) - .connectionTimeoutMs((int) ZK_CONNECTION_TIMEOUT.toMillis()) - .connectString(connectionSpec) - .zookeeperFactory(new VespaZooKeeperFactory(createClientConfig(clientConfigFile))) - .dontUseContainerParents() // TODO: Remove when we know ZooKeeper 3.5 works fine, consider waiting until Vespa 8 - .build()); - } - - protected Curator(String connectionSpec, - String zooKeeperEnsembleConnectionSpec, - Function<RetryPolicy, CuratorFramework> curatorFactory) { - this(connectionSpec, zooKeeperEnsembleConnectionSpec, curatorFactory, - new ExponentialBackoffRetry((int) BASE_SLEEP_TIME.toMillis(), MAX_RETRIES)); + protected Curator(String connectionSpec, String zooKeeperEnsembleConnectionSpec, Function<RetryPolicy, CuratorFramework> curatorFactory) { + this(ConnectionSpec.create(connectionSpec, zooKeeperEnsembleConnectionSpec), curatorFactory.apply(DEFAULT_RETRY_POLICY)); } - private Curator(String connectionSpec, - String zooKeeperEnsembleConnectionSpec, - Function<RetryPolicy, CuratorFramework> curatorFactory, - RetryPolicy retryPolicy) { - this.connectionSpec = connectionSpec; - this.retryPolicy = retryPolicy; - this.curatorFramework = curatorFactory.apply(retryPolicy); - if (this.curatorFramework != null) { - validateConnectionSpec(connectionSpec); - validateConnectionSpec(zooKeeperEnsembleConnectionSpec); - addLoggingListener(); - curatorFramework.start(); - } - - this.zooKeeperEnsembleConnectionSpec = zooKeeperEnsembleConnectionSpec; - this.zooKeeperEnsembleCount = zooKeeperEnsembleConnectionSpec.split(",").length; + Curator(ConnectionSpec connectionSpec, Optional<File> clientConfigFile) { + this(connectionSpec, + CuratorFrameworkFactory + .builder() + .retryPolicy(DEFAULT_RETRY_POLICY) + .sessionTimeoutMs((int) ZK_SESSION_TIMEOUT.toMillis()) + .connectionTimeoutMs((int) ZK_CONNECTION_TIMEOUT.toMillis()) + .connectString(connectionSpec.local()) + .zookeeperFactory(new VespaZooKeeperFactory(createClientConfig(clientConfigFile))) + .dontUseContainerParents() // TODO: Remove when we know ZooKeeper 3.5 works fine, consider waiting until Vespa 8 + .build()); } - private static String createConnectionSpec(ConfigserverConfig configserverConfig) { - return configserverConfig.zookeeperLocalhostAffinity() - ? createConnectionSpecForLocalhost(configserverConfig) - : createEnsembleConnectionSpec(configserverConfig); + private Curator(ConnectionSpec connectionSpec, CuratorFramework curatorFramework) { + this.connectionSpec = Objects.requireNonNull(connectionSpec); + this.curatorFramework = Objects.requireNonNull(curatorFramework); + addLoggingListener(); + curatorFramework.start(); } private static ZKClientConfig createClientConfig(Optional<File> clientConfigFile) { @@ -148,39 +135,6 @@ public class Curator implements AutoCloseable { } } - private static String createEnsembleConnectionSpec(ConfigserverConfig config) { - StringBuilder connectionSpec = new StringBuilder(); - for (int i = 0; i < config.zookeeperserver().size(); i++) { - if (connectionSpec.length() > 0) { - connectionSpec.append(','); - } - ConfigserverConfig.Zookeeperserver server = config.zookeeperserver(i); - connectionSpec.append(server.hostname()); - connectionSpec.append(':'); - connectionSpec.append(server.port()); - } - return connectionSpec.toString(); - } - - static String createConnectionSpecForLocalhost(ConfigserverConfig config) { - String thisServer = HostName.getLocalhost(); - - for (int i = 0; i < config.zookeeperserver().size(); i++) { - ConfigserverConfig.Zookeeperserver server = config.zookeeperserver(i); - if (thisServer.equals(server.hostname())) { - return String.format("%s:%d", server.hostname(), server.port()); - } - } - - throw new IllegalArgumentException("Unable to create connect string to localhost: " + - "There is no localhost server specified in config: " + config); - } - - private static void validateConnectionSpec(String connectionSpec) { - if (connectionSpec == null || connectionSpec.isEmpty()) - throw new IllegalArgumentException(String.format("Connections spec '%s' is not valid", connectionSpec)); - } - /** * Returns the ZooKeeper "connect string" used by curator: a comma-separated list of * host:port of ZooKeeper endpoints to connect to. This may be a subset of @@ -189,7 +143,7 @@ public class Curator implements AutoCloseable { * * This may be empty but never null */ - public String connectionSpec() { return connectionSpec; } + public String connectionSpec() { return connectionSpec.local(); } /** For internal use; prefer creating a {@link CuratorCounter} */ public DistributedAtomicLong createAtomicCounter(String path) { @@ -243,13 +197,14 @@ public class Curator implements AutoCloseable { * A convenience method which sets some content at a path. * If the path and any of its parents does not exists they are created. */ + // TODO: Use create().orSetData() in Curator 4 and later public void set(Path path, byte[] data) { + if ( ! exists(path)) + create(path); + String absolutePath = path.getAbsolute(); try { - if ( ! exists(path)) - framework().create().creatingParentsIfNeeded().forPath(absolutePath, data); - else - framework().setData().forPath(absolutePath, data); + framework().setData().forPath(absolutePath, data); } catch (Exception e) { throw new RuntimeException("Could not set data at " + absolutePath, e); } @@ -432,7 +387,7 @@ public class Curator implements AutoCloseable { * TODO: Move method out of this class. */ public String zooKeeperEnsembleConnectionSpec() { - return zooKeeperEnsembleConnectionSpec; + return connectionSpec.ensemble(); } /** @@ -440,7 +395,7 @@ public class Curator implements AutoCloseable { * WARNING: This may be different from the number of servers this Curator may connect to. * TODO: Move method out of this class. */ - public int zooKeeperEnsembleCount() { return zooKeeperEnsembleCount; } + public int zooKeeperEnsembleCount() { return connectionSpec.ensembleSize(); } private static Optional<String> getEnvironmentVariable(String variableName) { return Optional.ofNullable(System.getenv().get(variableName)) diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java index 3da7678c44e..26f1c336874 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java @@ -1,94 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator.mock; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; -import com.yahoo.collections.Pair; -import com.yahoo.concurrent.Lock; -import com.yahoo.concurrent.Locks; import com.yahoo.path.Path; -import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.recipes.CuratorLockException; -import org.apache.curator.CuratorZookeeperClient; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.api.ACLBackgroundPathAndBytesable; -import org.apache.curator.framework.api.ACLCreateModeBackgroundPathAndBytesable; -import org.apache.curator.framework.api.ACLCreateModePathAndBytesable; -import org.apache.curator.framework.api.ACLPathAndBytesable; -import org.apache.curator.framework.api.BackgroundCallback; -import org.apache.curator.framework.api.BackgroundPathAndBytesable; -import org.apache.curator.framework.api.BackgroundPathable; -import org.apache.curator.framework.api.BackgroundVersionable; -import org.apache.curator.framework.api.ChildrenDeletable; -import org.apache.curator.framework.api.CreateBackgroundModeACLable; -import org.apache.curator.framework.api.CreateBuilder; -import org.apache.curator.framework.api.CuratorListener; -import org.apache.curator.framework.api.CuratorWatcher; -import org.apache.curator.framework.api.DeleteBuilder; -import org.apache.curator.framework.api.ErrorListenerPathAndBytesable; -import org.apache.curator.framework.api.ErrorListenerPathable; -import org.apache.curator.framework.api.ExistsBuilder; -import org.apache.curator.framework.api.ExistsBuilderMain; -import org.apache.curator.framework.api.GetACLBuilder; -import org.apache.curator.framework.api.GetChildrenBuilder; -import org.apache.curator.framework.api.GetDataBuilder; -import org.apache.curator.framework.api.GetDataWatchBackgroundStatable; -import org.apache.curator.framework.api.PathAndBytesable; -import org.apache.curator.framework.api.Pathable; -import org.apache.curator.framework.api.ProtectACLCreateModePathAndBytesable; -import org.apache.curator.framework.api.SetACLBuilder; -import org.apache.curator.framework.api.SetDataBackgroundVersionable; -import org.apache.curator.framework.api.SetDataBuilder; -import org.apache.curator.framework.api.SyncBuilder; -import org.apache.curator.framework.api.UnhandledErrorListener; -import org.apache.curator.framework.api.VersionPathAndBytesable; -import org.apache.curator.framework.api.WatchPathable; -import org.apache.curator.framework.api.Watchable; -import org.apache.curator.framework.api.transaction.CuratorTransaction; -import org.apache.curator.framework.api.transaction.CuratorTransactionBridge; -import org.apache.curator.framework.api.transaction.CuratorTransactionFinal; -import org.apache.curator.framework.api.transaction.CuratorTransactionResult; -import org.apache.curator.framework.api.transaction.TransactionCheckBuilder; -import org.apache.curator.framework.api.transaction.TransactionCreateBuilder; -import org.apache.curator.framework.api.transaction.TransactionDeleteBuilder; -import org.apache.curator.framework.api.transaction.TransactionSetDataBuilder; -import org.apache.curator.framework.imps.CuratorFrameworkState; -import org.apache.curator.framework.listen.Listenable; -import org.apache.curator.framework.recipes.atomic.AtomicStats; -import org.apache.curator.framework.recipes.atomic.AtomicValue; import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong; -import org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.curator.framework.recipes.cache.NodeCacheListener; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessLock; -import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex; -import org.apache.curator.framework.state.ConnectionStateListener; -import org.apache.curator.utils.EnsurePath; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Stat; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import static com.yahoo.vespa.curator.mock.MemoryFileSystem.Node; /** * <p>A <b>non thread safe</b> mock of the curator API. @@ -105,24 +25,7 @@ import static com.yahoo.vespa.curator.mock.MemoryFileSystem.Node; */ public class MockCurator extends Curator { - public boolean timeoutOnLock = false; - public boolean throwExceptionOnLock = false; - private boolean shouldTimeoutOnEnter = false; - private int monotonicallyIncreasingNumber = 0; - private final boolean stableOrdering; private String zooKeeperEnsembleConnectionSpec = ""; - private final Locks<String> locks = new Locks<>(Long.MAX_VALUE, TimeUnit.DAYS); - - /** The file system used by this mock to store zookeeper files and directories */ - private final MemoryFileSystem fileSystem = new MemoryFileSystem(); - - /** Atomic counters. A more accurate mock would store these as files in the file system */ - private final Map<String, MockAtomicCounter> atomicCounters = new ConcurrentHashMap<>(); - - /** Listeners to changes to a particular path */ - private final ListenerMap listeners = new ListenerMap(); - - private final CuratorFramework curatorFramework; /** Creates a mock curator with stable ordering */ @Inject @@ -137,26 +40,24 @@ public class MockCurator extends Curator { * This is not what ZooKeeper does. */ public MockCurator(boolean stableOrdering) { - super("", "", (retryPolicy) -> null); - this.stableOrdering = stableOrdering; - curatorFramework = new MockCuratorFramework(); - curatorFramework.start(); + super("host1:2181", "host1:2181", (retryPolicy) -> new MockCuratorFramework(stableOrdering, false)); + } + + private MockCuratorFramework mockFramework() { + return (MockCuratorFramework) super.framework(); } /** * Lists the entire content of this curator instance as a multiline string. * Useful for debugging. */ - public String dumpState() { return fileSystem.dumpState(); } - - /** Returns a started curator framework */ - public CuratorFramework framework() { return curatorFramework; } + public String dumpState() { return mockFramework().fileSystem().dumpState(); } /** Returns an atomic counter in this, or empty if no such counter is created */ public Optional<DistributedAtomicLong> counter(String path) { - return Optional.ofNullable(atomicCounters.get(path)); + return Optional.ofNullable(mockFramework().atomicCounters().get(path)); } - + /** * Sets the ZooKeeper ensemble connection spec, which must be on the form * host1:port,host2:port ... @@ -170,1068 +71,37 @@ public class MockCurator extends Curator { return zooKeeperEnsembleConnectionSpec; } - // ----- Start of adaptor methods from Curator to the mock file system ----- - - /** Creates a node below the given directory root */ - private String createNode(String pathString, byte[] content, boolean createParents, CreateMode createMode, Node root, Listeners listeners) - throws KeeperException.NodeExistsException, KeeperException.NoNodeException { - validatePath(pathString); - Path path = Path.fromString(pathString); - if (path.isRoot()) return "/"; // the root already exists - Node parent = root.getNode(Paths.get(path.getParentPath().toString()), createParents); - String name = nodeName(path.getName(), createMode); - - if (parent == null) - throw new KeeperException.NoNodeException(path.getParentPath().toString()); - if (parent.children().containsKey(path.getName())) - throw new KeeperException.NodeExistsException(path.toString()); - - parent.add(name).setContent(content); - String nodePath = "/" + path.getParentPath().toString() + "/" + name; - listeners.notify(Path.fromString(nodePath), content, PathChildrenCacheEvent.Type.CHILD_ADDED); - return nodePath; - } - - /** Deletes a node below the given directory root */ - private void deleteNode(String pathString, boolean deleteChildren, Node root, Listeners listeners) - throws KeeperException.NoNodeException, KeeperException.NotEmptyException { - validatePath(pathString); - Path path = Path.fromString(pathString); - Node parent = root.getNode(Paths.get(path.getParentPath().toString()), false); - if (parent == null) throw new KeeperException.NoNodeException(path.toString()); - Node node = parent.children().get(path.getName()); - if (node == null) throw new KeeperException.NoNodeException(path.getName() + " under " + parent); - if ( ! node.children().isEmpty() && ! deleteChildren) - throw new KeeperException.NotEmptyException(path.toString()); - parent.remove(path.getName()); - listeners.notify(path, new byte[0], PathChildrenCacheEvent.Type.CHILD_REMOVED); - } - - /** Returns the data of a node */ - private byte[] getData(String pathString, Node root) throws KeeperException.NoNodeException { - validatePath(pathString); - return getNode(pathString, root).getContent(); - } - - /** sets the data of an existing node */ - private void setData(String pathString, byte[] content, Node root, Listeners listeners) - throws KeeperException.NoNodeException { - validatePath(pathString); - getNode(pathString, root).setContent(content); - listeners.notify(Path.fromString(pathString), content, PathChildrenCacheEvent.Type.CHILD_UPDATED); - } - - private List<String> getChildren(String path, Node root) throws KeeperException.NoNodeException { - validatePath(path); - Node node = root.getNode(Paths.get(path), false); - if (node == null) throw new KeeperException.NoNodeException(path); - List<String> children = new ArrayList<>(node.children().keySet()); - if (! stableOrdering) - Collections.shuffle(children); - return children; - } - - private boolean exists(String path, Node root) { - validatePath(path); - Node parent = root.getNode(Paths.get(Path.fromString(path).getParentPath().toString()), false); - if (parent == null) return false; - Node node = parent.children().get(Path.fromString(path).getName()); - return node != null; - } - - /** Returns a node or throws the appropriate exception if it doesn't exist */ - private Node getNode(String pathString, Node root) throws KeeperException.NoNodeException { - validatePath(pathString); - Path path = Path.fromString(pathString); - Node parent = root.getNode(Paths.get(path.getParentPath().toString()), false); - if (parent == null) throw new KeeperException.NoNodeException(path.toString()); - Node node = parent.children().get(path.getName()); - if (node == null) throw new KeeperException.NoNodeException(path.toString()); - return node; - } - - private String nodeName(String baseName, CreateMode createMode) { - switch (createMode) { - case PERSISTENT: case EPHEMERAL: return baseName; - case PERSISTENT_SEQUENTIAL: case EPHEMERAL_SEQUENTIAL: return baseName + monotonicallyIncreasingNumber++; - default: throw new UnsupportedOperationException(createMode + " support not implemented in MockCurator"); - } - } - - /** Validates a path using the same rules as ZooKeeper */ - public static String validatePath(String path) throws IllegalArgumentException { - if (path == null) throw new IllegalArgumentException("Path cannot be null"); - if (path.length() == 0) throw new IllegalArgumentException("Path length must be > 0"); - if (path.charAt(0) != '/') throw new IllegalArgumentException("Path must start with / character"); - if (path.length() == 1) return path; // done checking - it's the root - if (path.charAt(path.length() - 1) == '/') - throw new IllegalArgumentException("Path must not end with / character"); - - String reason = null; - char lastc = '/'; - char chars[] = path.toCharArray(); - char c; - for (int i = 1; i < chars.length; lastc = chars[i], i++) { - c = chars[i]; - - if (c == 0) { - reason = "null character not allowed @" + i; - break; - } else if (c == '/' && lastc == '/') { - reason = "empty node name specified @" + i; - break; - } else if (c == '.' && lastc == '.') { - if (chars[i-2] == '/' && ((i + 1 == chars.length) || chars[i+1] == '/')) { - reason = "relative paths not allowed @" + i; - break; - } - } else if (c == '.') { - if (chars[i-1] == '/' && ((i + 1 == chars.length) || chars[i+1] == '/')) { - reason = "relative paths not allowed @" + i; - break; - } - } else if (c > '\u0000' && c < '\u001f' || c > '\u007f' && c < '\u009F' - || c > '\ud800' && c < '\uf8ff' || c > '\ufff0' && c < '\uffff') { - reason = "invalid charater @" + i; - break; - } - } - - if (reason != null) - throw new IllegalArgumentException("Invalid path string \"" + path + "\" caused by " + reason); - return path; - } - - // ----- Mock of Curator recipes accessed through our Curator interface ----- - @Override public DistributedAtomicLong createAtomicCounter(String path) { - MockAtomicCounter counter = atomicCounters.get(path); - if (counter == null) { - counter = new MockAtomicCounter(path); - atomicCounters.put(path, counter); - } - return counter; + return mockFramework().createAtomicCounter(path); } - /** Create a mutex which ensures exclusive access within this single vm */ @Override public InterProcessLock createMutex(String path) { - return new MockLock(path); - } - - public MockCurator timeoutBarrierOnEnter(boolean shouldTimeout) { - shouldTimeoutOnEnter = shouldTimeout; - return this; + return mockFramework().createMutex(path); } @Override public CompletionWaiter getCompletionWaiter(Path parentPath, int numMembers, String id) { - return new MockCompletionWaiter(); + return mockFramework().createCompletionWaiter(); } @Override public CompletionWaiter createCompletionWaiter(Path parentPath, String waiterNode, int numMembers, String id) { - return new MockCompletionWaiter(); + return mockFramework().createCompletionWaiter(); } @Override public DirectoryCache createDirectoryCache(String path, boolean cacheData, boolean dataIsCompressed, ExecutorService executorService) { - return new MockDirectoryCache(Path.fromString(path)); + return mockFramework().createDirectoryCache(path); } @Override public FileCache createFileCache(String path, boolean dataIsCompressed) { - return new MockFileCache(Path.fromString(path)); + return mockFramework().createFileCache(path); } @Override public int zooKeeperEnsembleCount() { return 1; } - /** - * Invocation of changes to the file system state is abstracted through this to allow transactional - * changes to notify on commit - */ - private abstract class Listeners { - - /** Translating method */ - public final void notify(Path path, byte[] data, PathChildrenCacheEvent.Type type) { - String pathString = "/" + path.toString(); // this silly path class strips the leading "/" :-/ - PathChildrenCacheEvent event = new PathChildrenCacheEvent(type, new ChildData(pathString, null, data)); - notify(path, event); - } - - public abstract void notify(Path path, PathChildrenCacheEvent event); - - } - - /** The regular listener implementation which notifies registered file and directory listeners */ - private class ListenerMap extends Listeners { - - private final Map<Path, PathChildrenCacheListener> directoryListeners = new ConcurrentHashMap<>(); - private final Map<Path, NodeCacheListener> fileListeners = new ConcurrentHashMap<>(); - - public void add(Path path, PathChildrenCacheListener listener) { - directoryListeners.put(path, listener); - } - - public void add(Path path, NodeCacheListener listener) { - fileListeners.put(path, listener); - } - - @Override - public void notify(Path path, PathChildrenCacheEvent event) { - try { - // Snapshot directoryListeners in case notification leads to new directoryListeners added - Set<Map.Entry<Path, PathChildrenCacheListener>> directoryListenerSnapshot = new HashSet<>(directoryListeners.entrySet()); - for (Map.Entry<Path, PathChildrenCacheListener> listener : directoryListenerSnapshot) { - if (path.isChildOf(listener.getKey())) - listener.getValue().childEvent(curatorFramework, event); - } - - // Snapshot directoryListeners in case notification leads to new directoryListeners added - Set<Map.Entry<Path, NodeCacheListener>> fileListenerSnapshot = new HashSet<>(fileListeners.entrySet()); - for (Map.Entry<Path, NodeCacheListener> listener : fileListenerSnapshot) { - if (path.equals(listener.getKey())) - listener.getValue().nodeChanged(); - } - } - catch (Exception e) { - e.printStackTrace(); // TODO: Remove - throw new RuntimeException("Exception notifying listeners", e); - } - } - - } - - private class MockCompletionWaiter implements CompletionWaiter { - - @Override - public void awaitCompletion(Duration timeout) { - if (shouldTimeoutOnEnter) { - throw new CompletionTimeoutException(""); - } - } - - @Override - public void notifyCompletion() { - } - - } - - /** A lock which works inside a single vm */ - private class MockLock extends InterProcessSemaphoreMutex { - - private final String path; - - private Lock lock = null; - - public MockLock(String path) { - super(curatorFramework, path); - this.path = path; - } - - @Override - public boolean acquire(long timeout, TimeUnit unit) { - if (throwExceptionOnLock) - throw new CuratorLockException("Thrown by mock"); - if (timeoutOnLock) return false; - - try { - lock = locks.lock(path, timeout, unit); - return true; - } - catch (UncheckedTimeoutException e) { - return false; - } - } - - @Override - public void acquire() { - if (throwExceptionOnLock) - throw new CuratorLockException("Thrown by mock"); - - lock = locks.lock(path); - } - - @Override - public void release() { - if (lock != null) - lock.close(); - } - - } - - private class MockAtomicCounter extends DistributedAtomicLong { - - private boolean initialized = false; - private MockLongValue value = new MockLongValue(0); // yes, uninitialized returns 0 :-/ - - public MockAtomicCounter(String path) { - super(curatorFramework, path, retryPolicy); - } - - @Override - public boolean initialize(Long value) { - if (initialized) return false; - this.value = new MockLongValue(value); - initialized = true; - return true; - } - - @Override - public AtomicValue<Long> get() { - if (value == null) return new MockLongValue(0); - return value; - } - - public AtomicValue<Long> add(Long delta) throws Exception { - return trySet(value.postValue() + delta); - } - - public AtomicValue<Long> subtract(Long delta) throws Exception { - return trySet(value.postValue() - delta); - } - - @Override - public AtomicValue<Long> increment() { - return trySet(value.postValue() + 1); - } - - public AtomicValue<Long> decrement() throws Exception { - return trySet(value.postValue() - 1); - } - - @Override - public AtomicValue<Long> trySet(Long longval) { - value = new MockLongValue(longval); - return value; - } - - public void forceSet(Long newValue) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public AtomicValue<Long> compareAndSet(Long expectedValue, Long newValue) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - } - - private class MockLongValue implements AtomicValue<Long> { - - private AtomicLong value = new AtomicLong(); - - public MockLongValue(long value) { - this.value.set(value); - } - - @Override - public boolean succeeded() { - return true; - } - - public void setValue(long value) { - this.value.set(value); - } - - @Override - public Long preValue() { - return value.get(); - } - - @Override - public Long postValue() { - return value.get(); - } - - @Override - public AtomicStats getStats() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - } - - private class MockDirectoryCache implements DirectoryCache { - - /** The path this is caching and listening to */ - private Path path; - - public MockDirectoryCache(Path path) { - this.path = path; - } - - @Override - public void start() {} - - @Override - public void addListener(PathChildrenCacheListener listener) { - listeners.add(path, listener); - } - - @Override - public List<ChildData> getCurrentData() { - List<ChildData> childData = new ArrayList<>(); - for (String childName : getChildren(path)) { - Path childPath = path.append(childName); - childData.add(new ChildData(childPath.getAbsolute(), null, getData(childPath).get())); - } - return childData; - } - - @Override - public ChildData getCurrentData(Path fullPath) { - if (!fullPath.getParentPath().equals(path)) { - throw new IllegalArgumentException("Path '" + fullPath + "' is not a child path of '" + path + "'"); - } - - return getData(fullPath).map(bytes -> new ChildData(fullPath.getAbsolute(), null, bytes)).orElse(null); - } - - private void collectData(Node parent, Path parentPath, List<ChildData> data) { - for (Node child : parent.children().values()) { - Path childPath = parentPath.append(child.name()); - data.add(new ChildData("/" + childPath.toString(), null, child.getContent())); - } - } - - @Override - public void close() {} - - } - - private class MockFileCache implements FileCache { - - /** The path this is caching and listening to */ - private Path path; - - public MockFileCache(Path path) { - this.path = path; - } - - @Override - public void start() {} - - @Override - public void addListener(NodeCacheListener listener) { - listeners.add(path, listener); - } - - @Override - public ChildData getCurrentData() { - Node node = fileSystem.root().getNode(Paths.get(path.toString()), false); - if (node == null) return null; - return new ChildData("/" + path.toString(), null, node.getContent()); - } - - @Override - public void close() {} - - } - - // ----- The rest of this file is adapting the Curator (non-recipe) API to the ----- - // ----- file system methods above. ----- - // ----- There's nothing to see unless you are interested in an illustration of ----- - // ----- the folly of fluent API's or, more generally, mankind. ----- - - private abstract class MockBackgroundACLPathAndBytesableBuilder<T> implements PathAndBytesable<T>, ProtectACLCreateModePathAndBytesable<T> { - - public BackgroundPathAndBytesable<T> withACL(List<ACL> list) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public ACLBackgroundPathAndBytesable<T> withMode(CreateMode createMode) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ACLCreateModeBackgroundPathAndBytesable<String> withProtection() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public T forPath(String s, byte[] bytes) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public T forPath(String s) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - } - - private class MockCreateBuilder extends MockBackgroundACLPathAndBytesableBuilder<String> implements CreateBuilder { - - private boolean createParents = false; - private CreateMode createMode = CreateMode.PERSISTENT; - - @Override - public ProtectACLCreateModePathAndBytesable<String> creatingParentsIfNeeded() { - createParents = true; - return this; - } - - @Override - public ACLCreateModeBackgroundPathAndBytesable<String> withProtection() { - // Protection against the server crashing after creating the file but before returning to the client. - // Not relevant for an in-memory mock, obviously - return this; - } - - public ACLBackgroundPathAndBytesable<String> withMode(CreateMode createMode) { - this.createMode = createMode; - return this; - } - - @Override - public CreateBackgroundModeACLable compressed() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ProtectACLCreateModePathAndBytesable<String> creatingParentContainersIfNeeded() { - // TODO: Add proper support for container nodes, see https://issues.apache.org/jira/browse/ZOOKEEPER-2163. - return creatingParentsIfNeeded(); - } - - @Override - @Deprecated - public ACLPathAndBytesable<String> withProtectedEphemeralSequential() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public String forPath(String s) throws Exception { - return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); - } - - public String forPath(String s, byte[] bytes) throws Exception { - return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - } - - private class MockBackgroundPathableBuilder<T> implements BackgroundPathable<T>, Watchable<BackgroundPathable<T>> { - - @Override - public ErrorListenerPathable<T> inBackground() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<T> inBackground(Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public T forPath(String s) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public BackgroundPathable<T> watched() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public BackgroundPathable<T> usingWatcher(Watcher watcher) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public BackgroundPathable<T> usingWatcher(CuratorWatcher curatorWatcher) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - } - - private class MockGetChildrenBuilder extends MockBackgroundPathableBuilder<List<String>> implements GetChildrenBuilder { - - @Override - public WatchPathable<List<String>> storingStatIn(Stat stat) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public List<String> forPath(String path) throws Exception { - return getChildren(path, fileSystem.root()); - } - - } - - private class MockExistsBuilder extends MockBackgroundPathableBuilder<Stat> implements ExistsBuilder { - - @Override - public Stat forPath(String path) throws Exception { - try { - Node node = getNode(path, fileSystem.root()); - Stat stat = new Stat(); - stat.setVersion(node.version()); - return stat; - } - catch (KeeperException.NoNodeException e) { - return null; - } - } - - @Override - public ExistsBuilderMain creatingParentContainersIfNeeded() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - } - - private class MockDeleteBuilder extends MockBackgroundPathableBuilder<Void> implements DeleteBuilder { - - private boolean deleteChildren = false; - - @Override - public BackgroundVersionable deletingChildrenIfNeeded() { - deleteChildren = true; - return this; - } - - @Override - public ChildrenDeletable guaranteed() { - return this; - } - - @Override - public BackgroundPathable<Void> withVersion(int i) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public Void forPath(String pathString) throws Exception { - deleteNode(pathString, deleteChildren, fileSystem.root(), listeners); - return null; - } - - } - - private class MockGetDataBuilder extends MockBackgroundPathableBuilder<byte[]> implements GetDataBuilder { - - @Override - public GetDataWatchBackgroundStatable decompressed() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public WatchPathable<byte[]> storingStatIn(Stat stat) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - public byte[] forPath(String path) throws Exception { - return getData(path, fileSystem.root()); - } - - } - - private class MockSetDataBuilder extends MockBackgroundACLPathAndBytesableBuilder<Stat> implements SetDataBuilder { - - @Override - public SetDataBackgroundVersionable compressed() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public BackgroundPathAndBytesable<Stat> withVersion(int i) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public Stat forPath(String path, byte[] bytes) throws Exception { - setData(path, bytes, fileSystem.root(), listeners); - return null; - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground(Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Object o) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - } - - /** Allows addition of directoryListeners which are never called */ - private class MockListenable<T> implements Listenable<T> { - - @Override - public void addListener(T t) { - } - - @Override - public void addListener(T t, Executor executor) { - } - - @Override - public void removeListener(T t) { - } - - } - - private class MockCuratorTransactionFinal implements CuratorTransactionFinal { - - /** The new directory root in which the transactional changes are made */ - private Node newRoot; - - private boolean committed = false; - - private final DelayedListener delayedListener = new DelayedListener(); - - public MockCuratorTransactionFinal() { - newRoot = fileSystem.root().clone(); - } - - @Override - public Collection<CuratorTransactionResult> commit() throws Exception { - fileSystem.replaceRoot(newRoot); - committed = true; - delayedListener.commit(); - return null; // TODO - } - - @Override - public TransactionCreateBuilder create() { - ensureNotCommitted(); - return new MockTransactionCreateBuilder(); - } - - @Override - public TransactionDeleteBuilder delete() { - ensureNotCommitted(); - return new MockTransactionDeleteBuilder(); - } - - @Override - public TransactionSetDataBuilder setData() { - ensureNotCommitted(); - return new MockTransactionSetDataBuilder(); - } - - @Override - public TransactionCheckBuilder check() { - ensureNotCommitted(); - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - private void ensureNotCommitted() { - if (committed) throw new IllegalStateException("transaction already committed"); - } - - private class MockTransactionCreateBuilder implements TransactionCreateBuilder { - - private CreateMode createMode = CreateMode.PERSISTENT; - - @Override - public PathAndBytesable<CuratorTransactionBridge> withACL(List<ACL> list) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ACLCreateModePathAndBytesable<CuratorTransactionBridge> compressed() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public ACLPathAndBytesable<CuratorTransactionBridge> withMode(CreateMode createMode) { - this.createMode = createMode; - return this; - } - - @Override - public CuratorTransactionBridge forPath(String s, byte[] bytes) throws Exception { - createNode(s, bytes, false, createMode, newRoot, delayedListener); - return new MockCuratorTransactionBridge(); - } - - @Override - public CuratorTransactionBridge forPath(String s) throws Exception { - createNode(s, new byte[0], false, createMode, newRoot, delayedListener); - return new MockCuratorTransactionBridge(); - } - - } - - private class MockTransactionDeleteBuilder implements TransactionDeleteBuilder { - - @Override - public Pathable<CuratorTransactionBridge> withVersion(int i) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public CuratorTransactionBridge forPath(String path) throws Exception { - deleteNode(path, false, newRoot, delayedListener); - return new MockCuratorTransactionBridge(); - } - - } - - private class MockTransactionSetDataBuilder implements TransactionSetDataBuilder { - - @Override - public VersionPathAndBytesable<CuratorTransactionBridge> compressed() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public PathAndBytesable<CuratorTransactionBridge> withVersion(int i) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public CuratorTransactionBridge forPath(String s, byte[] bytes) throws Exception { - MockCurator.this.setData(s, bytes, newRoot, delayedListener); - return new MockCuratorTransactionBridge(); - } - - @Override - public CuratorTransactionBridge forPath(String s) throws Exception { - MockCurator.this.setData(s, new byte[0], newRoot, delayedListener); - return new MockCuratorTransactionBridge(); - } - - } - - private class MockCuratorTransactionBridge implements CuratorTransactionBridge { - - @Override - public CuratorTransactionFinal and() { - return MockCuratorTransactionFinal.this; - } - - } - - /** A class which collects listen events and forwards them to the regular directoryListeners on commit */ - private class DelayedListener extends Listeners { - - private final List<Pair<Path, PathChildrenCacheEvent>> events = new ArrayList<>(); - - @Override - public void notify(Path path, PathChildrenCacheEvent event) { - events.add(new Pair<>(path, event)); - } - - public void commit() { - for (Pair<Path, PathChildrenCacheEvent> event : events) - listeners.notify(event.getFirst(), event.getSecond()); - } - - } - - } - - private class MockCuratorFramework implements CuratorFramework { - - private CuratorFrameworkState curatorState = CuratorFrameworkState.LATENT; - - @Override - public void start() { - curatorState = CuratorFrameworkState.STARTED; - } - - @Override - public void close() { - curatorState = CuratorFrameworkState.STOPPED; - } - - @Override - public CuratorFrameworkState getState() { - return curatorState; - } - - @Override - @Deprecated - public boolean isStarted() { - return curatorState == CuratorFrameworkState.STARTED; - } - - @Override - public CreateBuilder create() { - return new MockCreateBuilder(); - } - - @Override - public DeleteBuilder delete() { - return new MockDeleteBuilder(); - } - - @Override - public ExistsBuilder checkExists() { - return new MockExistsBuilder(); - } - - @Override - public GetDataBuilder getData() { - return new MockGetDataBuilder(); - } - - @Override - public SetDataBuilder setData() { - return new MockSetDataBuilder(); - } - - @Override - public GetChildrenBuilder getChildren() { - return new MockGetChildrenBuilder(); - } - - @Override - public GetACLBuilder getACL() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public SetACLBuilder setACL() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public CuratorTransaction inTransaction() { - return new MockCuratorTransactionFinal(); - } - - @Override - @Deprecated - public void sync(String path, Object backgroundContextObject) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public void createContainers(String s) throws Exception { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public Listenable<ConnectionStateListener> getConnectionStateListenable() { - return new MockListenable<>(); - } - - @Override - public Listenable<CuratorListener> getCuratorListenable() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public Listenable<UnhandledErrorListener> getUnhandledErrorListenable() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - @Deprecated - public CuratorFramework nonNamespaceView() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public CuratorFramework usingNamespace(String newNamespace) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public String getNamespace() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public CuratorZookeeperClient getZookeeperClient() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Deprecated - @Override - public EnsurePath newNamespaceAwareEnsurePath(String path) { - return new EnsurePath(path); - } - - @Override - public void clearWatcherReferences(Watcher watcher) { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - @Override - public boolean blockUntilConnected(int i, TimeUnit timeUnit) throws InterruptedException { - return true; - } - - @Override - public void blockUntilConnected() throws InterruptedException { - - } - - @Override - public SyncBuilder sync() { - throw new UnsupportedOperationException("Not implemented in MockCurator"); - } - - } - } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java new file mode 100644 index 00000000000..9a845e56bfd --- /dev/null +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java @@ -0,0 +1,1169 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.mock; + +import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.collections.Pair; +import com.yahoo.concurrent.Lock; +import com.yahoo.concurrent.Locks; +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.CompletionTimeoutException; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.recipes.CuratorLockException; +import org.apache.curator.CuratorZookeeperClient; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.ACLBackgroundPathAndBytesable; +import org.apache.curator.framework.api.ACLCreateModeBackgroundPathAndBytesable; +import org.apache.curator.framework.api.ACLCreateModePathAndBytesable; +import org.apache.curator.framework.api.ACLPathAndBytesable; +import org.apache.curator.framework.api.BackgroundCallback; +import org.apache.curator.framework.api.BackgroundPathAndBytesable; +import org.apache.curator.framework.api.BackgroundPathable; +import org.apache.curator.framework.api.BackgroundVersionable; +import org.apache.curator.framework.api.ChildrenDeletable; +import org.apache.curator.framework.api.CreateBackgroundModeACLable; +import org.apache.curator.framework.api.CreateBuilder; +import org.apache.curator.framework.api.CuratorListener; +import org.apache.curator.framework.api.CuratorWatcher; +import org.apache.curator.framework.api.DeleteBuilder; +import org.apache.curator.framework.api.ErrorListenerPathAndBytesable; +import org.apache.curator.framework.api.ErrorListenerPathable; +import org.apache.curator.framework.api.ExistsBuilder; +import org.apache.curator.framework.api.ExistsBuilderMain; +import org.apache.curator.framework.api.GetACLBuilder; +import org.apache.curator.framework.api.GetChildrenBuilder; +import org.apache.curator.framework.api.GetDataBuilder; +import org.apache.curator.framework.api.GetDataWatchBackgroundStatable; +import org.apache.curator.framework.api.PathAndBytesable; +import org.apache.curator.framework.api.Pathable; +import org.apache.curator.framework.api.ProtectACLCreateModePathAndBytesable; +import org.apache.curator.framework.api.SetACLBuilder; +import org.apache.curator.framework.api.SetDataBackgroundVersionable; +import org.apache.curator.framework.api.SetDataBuilder; +import org.apache.curator.framework.api.SyncBuilder; +import org.apache.curator.framework.api.UnhandledErrorListener; +import org.apache.curator.framework.api.VersionPathAndBytesable; +import org.apache.curator.framework.api.WatchPathable; +import org.apache.curator.framework.api.Watchable; +import org.apache.curator.framework.api.transaction.CuratorTransaction; +import org.apache.curator.framework.api.transaction.CuratorTransactionBridge; +import org.apache.curator.framework.api.transaction.CuratorTransactionFinal; +import org.apache.curator.framework.api.transaction.CuratorTransactionResult; +import org.apache.curator.framework.api.transaction.TransactionCheckBuilder; +import org.apache.curator.framework.api.transaction.TransactionCreateBuilder; +import org.apache.curator.framework.api.transaction.TransactionDeleteBuilder; +import org.apache.curator.framework.api.transaction.TransactionSetDataBuilder; +import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.framework.listen.Listenable; +import org.apache.curator.framework.recipes.atomic.AtomicStats; +import org.apache.curator.framework.recipes.atomic.AtomicValue; +import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.NodeCacheListener; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import org.apache.curator.framework.recipes.locks.InterProcessLock; +import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex; +import org.apache.curator.framework.state.ConnectionStateListener; +import org.apache.curator.retry.RetryForever; +import org.apache.curator.utils.EnsurePath; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; + +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A mock implementation of{@link CuratorFramework} for testing purposes. + * + * @author mpolden + */ +public class MockCuratorFramework implements CuratorFramework { + + private final boolean shouldTimeoutOnEnter; + private final boolean stableOrdering; + private final Locks<String> locks = new Locks<>(Long.MAX_VALUE, TimeUnit.DAYS); + + /** The file system used by this mock to store zookeeper files and directories */ + private final MemoryFileSystem fileSystem = new MemoryFileSystem(); + + /** Atomic counters. A more accurate mock would store these as files in the file system */ + private final Map<String, MockAtomicCounter> atomicCounters = new ConcurrentHashMap<>(); + + /** Listeners to changes to a particular path */ + private final ListenerMap listeners = new ListenerMap(); + + private CuratorFrameworkState curatorState = CuratorFrameworkState.LATENT; + private int monotonicallyIncreasingNumber = 0; + + public MockCuratorFramework(boolean stableOrdering, boolean shouldTimeoutOnEnter) { + this.stableOrdering = stableOrdering; + this.shouldTimeoutOnEnter = shouldTimeoutOnEnter; + } + + public Map<String, MockAtomicCounter> atomicCounters() { + return Collections.unmodifiableMap(atomicCounters); + } + + public MemoryFileSystem fileSystem() { + return fileSystem; + } + + @Override + public void start() { + curatorState = CuratorFrameworkState.STARTED; + } + + @Override + public void close() { + curatorState = CuratorFrameworkState.STOPPED; + } + + @Override + public CuratorFrameworkState getState() { + return curatorState; + } + + @Override + @Deprecated + public boolean isStarted() { + return curatorState == CuratorFrameworkState.STARTED; + } + + @Override + public CreateBuilder create() { + return new MockCreateBuilder(); + } + + @Override + public DeleteBuilder delete() { + return new MockDeleteBuilder(); + } + + @Override + public ExistsBuilder checkExists() { + return new MockExistsBuilder(); + } + + @Override + public GetDataBuilder getData() { + return new MockGetDataBuilder(); + } + + @Override + public SetDataBuilder setData() { + return new MockSetDataBuilder(); + } + + @Override + public GetChildrenBuilder getChildren() { + return new MockGetChildrenBuilder(); + } + + @Override + public GetACLBuilder getACL() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public SetACLBuilder setACL() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public CuratorTransaction inTransaction() { + return new MockCuratorTransactionFinal(); + } + + @Override + @Deprecated + public void sync(String path, Object backgroundContextObject) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public void createContainers(String s) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public Listenable<ConnectionStateListener> getConnectionStateListenable() { + return new MockListenable<>(); + } + + @Override + public Listenable<CuratorListener> getCuratorListenable() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public Listenable<UnhandledErrorListener> getUnhandledErrorListenable() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + @Deprecated + public CuratorFramework nonNamespaceView() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public CuratorFramework usingNamespace(String newNamespace) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public String getNamespace() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public CuratorZookeeperClient getZookeeperClient() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Deprecated + @Override + public EnsurePath newNamespaceAwareEnsurePath(String path) { + return new EnsurePath(path); + } + + @Override + public void clearWatcherReferences(Watcher watcher) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public boolean blockUntilConnected(int i, TimeUnit timeUnit) { + return true; + } + + @Override + public void blockUntilConnected() { + } + + @Override + public SyncBuilder sync() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + // ----- Factory methods for mocks */ + + public InterProcessLock createMutex(String path) { + return new MockCuratorFramework.MockLock(path); + } + + public MockAtomicCounter createAtomicCounter(String path) { + return atomicCounters.computeIfAbsent(path, (k) -> new MockAtomicCounter(path)); + } + + public Curator.CompletionWaiter createCompletionWaiter() { + return new MockCuratorFramework.MockCompletionWaiter(); + } + + public Curator.DirectoryCache createDirectoryCache(String path) { + return new MockDirectoryCache(Path.fromString(path)); + } + + public Curator.FileCache createFileCache(String path) { + return new MockFileCache(Path.fromString(path)); + } + + // ----- Start of adaptor methods from Curator to the mock file system ----- + + /** Creates a node below the given directory root */ + private String createNode(String pathString, byte[] content, boolean createParents, CreateMode createMode, MemoryFileSystem.Node root, Listeners listeners) + throws KeeperException.NodeExistsException, KeeperException.NoNodeException { + validatePath(pathString); + Path path = Path.fromString(pathString); + if (path.isRoot()) return "/"; // the root already exists + MemoryFileSystem.Node parent = root.getNode(Paths.get(path.getParentPath().toString()), createParents); + String name = nodeName(path.getName(), createMode); + + if (parent == null) + throw new KeeperException.NoNodeException(path.getParentPath().toString()); + if (parent.children().containsKey(path.getName())) + throw new KeeperException.NodeExistsException(path.toString()); + + parent.add(name).setContent(content); + String nodePath = "/" + path.getParentPath().toString() + "/" + name; + listeners.notify(Path.fromString(nodePath), content, PathChildrenCacheEvent.Type.CHILD_ADDED); + return nodePath; + } + + /** Deletes a node below the given directory root */ + private void deleteNode(String pathString, boolean deleteChildren, MemoryFileSystem.Node root, Listeners listeners) + throws KeeperException.NoNodeException, KeeperException.NotEmptyException { + validatePath(pathString); + Path path = Path.fromString(pathString); + MemoryFileSystem.Node parent = root.getNode(Paths.get(path.getParentPath().toString()), false); + if (parent == null) throw new KeeperException.NoNodeException(path.toString()); + MemoryFileSystem.Node node = parent.children().get(path.getName()); + if (node == null) throw new KeeperException.NoNodeException(path.getName() + " under " + parent); + if ( ! node.children().isEmpty() && ! deleteChildren) + throw new KeeperException.NotEmptyException(path.toString()); + parent.remove(path.getName()); + listeners.notify(path, new byte[0], PathChildrenCacheEvent.Type.CHILD_REMOVED); + } + + /** Returns the data of a node */ + private byte[] getData(String pathString, MemoryFileSystem.Node root) throws KeeperException.NoNodeException { + validatePath(pathString); + return getNode(pathString, root).getContent(); + } + + /** sets the data of an existing node */ + private void setData(String pathString, byte[] content, MemoryFileSystem.Node root, Listeners listeners) + throws KeeperException.NoNodeException { + validatePath(pathString); + getNode(pathString, root).setContent(content); + listeners.notify(Path.fromString(pathString), content, PathChildrenCacheEvent.Type.CHILD_UPDATED); + } + + private List<String> getChildren(String path, MemoryFileSystem.Node root) throws KeeperException.NoNodeException { + validatePath(path); + MemoryFileSystem.Node node = root.getNode(Paths.get(path), false); + if (node == null) throw new KeeperException.NoNodeException(path); + List<String> children = new ArrayList<>(node.children().keySet()); + if (! stableOrdering) + Collections.shuffle(children); + return children; + } + + /** Returns a node or throws the appropriate exception if it doesn't exist */ + private MemoryFileSystem.Node getNode(String pathString, MemoryFileSystem.Node root) throws KeeperException.NoNodeException { + validatePath(pathString); + Path path = Path.fromString(pathString); + MemoryFileSystem.Node parent = root.getNode(Paths.get(path.getParentPath().toString()), false); + if (parent == null) throw new KeeperException.NoNodeException(path.toString()); + MemoryFileSystem.Node node = parent.children().get(path.getName()); + if (node == null) throw new KeeperException.NoNodeException(path.toString()); + return node; + } + + private String nodeName(String baseName, CreateMode createMode) { + switch (createMode) { + case PERSISTENT: case EPHEMERAL: return baseName; + case PERSISTENT_SEQUENTIAL: case EPHEMERAL_SEQUENTIAL: return baseName + monotonicallyIncreasingNumber++; + default: throw new UnsupportedOperationException(createMode + " support not implemented in MockCurator"); + } + } + + /** Validates a path using the same rules as ZooKeeper */ + public static String validatePath(String path) throws IllegalArgumentException { + if (path == null) throw new IllegalArgumentException("Path cannot be null"); + if (path.length() == 0) throw new IllegalArgumentException("Path length must be > 0"); + if (path.charAt(0) != '/') throw new IllegalArgumentException("Path must start with / character"); + if (path.length() == 1) return path; // done checking - it's the root + if (path.charAt(path.length() - 1) == '/') + throw new IllegalArgumentException("Path must not end with / character"); + + String reason = null; + char lastc = '/'; + char[] chars = path.toCharArray(); + char c; + for (int i = 1; i < chars.length; lastc = chars[i], i++) { + c = chars[i]; + + if (c == 0) { + reason = "null character not allowed @" + i; + break; + } else if (c == '/' && lastc == '/') { + reason = "empty node name specified @" + i; + break; + } else if (c == '.' && lastc == '.') { + if (chars[i-2] == '/' && ((i + 1 == chars.length) || chars[i+1] == '/')) { + reason = "relative paths not allowed @" + i; + break; + } + } else if (c == '.') { + if (chars[i-1] == '/' && ((i + 1 == chars.length) || chars[i+1] == '/')) { + reason = "relative paths not allowed @" + i; + break; + } + } else if (c > '\u0000' && c < '\u001f' || c > '\u007f' && c < '\u009F' + || c > '\ud800' && c < '\uf8ff' || c > '\ufff0' && c < '\uffff') { + reason = "invalid charater @" + i; + break; + } + } + + if (reason != null) + throw new IllegalArgumentException("Invalid path string \"" + path + "\" caused by " + reason); + return path; + } + + /** + * Invocation of changes to the file system state is abstracted through this to allow transactional + * changes to notify on commit + */ + private abstract static class Listeners { + + /** Translating method */ + public final void notify(Path path, byte[] data, PathChildrenCacheEvent.Type type) { + String pathString = "/" + path.toString(); // this silly path class strips the leading "/" :-/ + PathChildrenCacheEvent event = new PathChildrenCacheEvent(type, new ChildData(pathString, null, data)); + notify(path, event); + } + + public abstract void notify(Path path, PathChildrenCacheEvent event); + + } + + /** The regular listener implementation which notifies registered file and directory listeners */ + private class ListenerMap extends Listeners { + + private final Map<Path, PathChildrenCacheListener> directoryListeners = new ConcurrentHashMap<>(); + private final Map<Path, NodeCacheListener> fileListeners = new ConcurrentHashMap<>(); + + public void add(Path path, PathChildrenCacheListener listener) { + directoryListeners.put(path, listener); + } + + public void add(Path path, NodeCacheListener listener) { + fileListeners.put(path, listener); + } + + @Override + public void notify(Path path, PathChildrenCacheEvent event) { + try { + // Snapshot directoryListeners in case notification leads to new directoryListeners added + Set<Map.Entry<Path, PathChildrenCacheListener>> directoryListenerSnapshot = new HashSet<>(directoryListeners.entrySet()); + for (Map.Entry<Path, PathChildrenCacheListener> listener : directoryListenerSnapshot) { + if (path.isChildOf(listener.getKey())) + listener.getValue().childEvent(MockCuratorFramework.this, event); + } + + // Snapshot directoryListeners in case notification leads to new directoryListeners added + Set<Map.Entry<Path, NodeCacheListener>> fileListenerSnapshot = new HashSet<>(fileListeners.entrySet()); + for (Map.Entry<Path, NodeCacheListener> listener : fileListenerSnapshot) { + if (path.equals(listener.getKey())) + listener.getValue().nodeChanged(); + } + } + catch (Exception e) { + e.printStackTrace(); // TODO: Remove + throw new RuntimeException("Exception notifying listeners", e); + } + } + + } + + private class MockCompletionWaiter implements Curator.CompletionWaiter { + + @Override + public void awaitCompletion(Duration timeout) { + if (shouldTimeoutOnEnter) { + throw new CompletionTimeoutException(""); + } + } + + @Override + public void notifyCompletion() { + } + + } + + /** A lock which works inside a single vm */ + private class MockLock extends InterProcessSemaphoreMutex { + + public boolean timeoutOnLock = false; + public boolean throwExceptionOnLock = false; + + private final String path; + + private Lock lock = null; + + public MockLock(String path) { + super(MockCuratorFramework.this, path); + this.path = path; + } + + @Override + public boolean acquire(long timeout, TimeUnit unit) { + if (throwExceptionOnLock) + throw new CuratorLockException("Thrown by mock"); + if (timeoutOnLock) return false; + + try { + lock = locks.lock(path, timeout, unit); + return true; + } + catch (UncheckedTimeoutException e) { + return false; + } + } + + @Override + public void acquire() { + if (throwExceptionOnLock) + throw new CuratorLockException("Thrown by mock"); + + lock = locks.lock(path); + } + + @Override + public void release() { + if (lock != null) + lock.close(); + } + + } + + private class MockAtomicCounter extends DistributedAtomicLong { + + private boolean initialized = false; + private MockLongValue value = new MockLongValue(0); // yes, uninitialized returns 0 :-/ + + public MockAtomicCounter(String path) { + super(MockCuratorFramework.this, path, new RetryForever(1_000)); + } + + @Override + public boolean initialize(Long value) { + if (initialized) return false; + this.value = new MockLongValue(value); + initialized = true; + return true; + } + + @Override + public AtomicValue<Long> get() { + if (value == null) return new MockLongValue(0); + return value; + } + + public AtomicValue<Long> add(Long delta) { + return trySet(value.postValue() + delta); + } + + public AtomicValue<Long> subtract(Long delta) { + return trySet(value.postValue() - delta); + } + + @Override + public AtomicValue<Long> increment() { + return trySet(value.postValue() + 1); + } + + public AtomicValue<Long> decrement() { + return trySet(value.postValue() - 1); + } + + @Override + public AtomicValue<Long> trySet(Long longval) { + value = new MockLongValue(longval); + return value; + } + + public void forceSet(Long newValue) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public AtomicValue<Long> compareAndSet(Long expectedValue, Long newValue) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + } + + private static class MockLongValue implements AtomicValue<Long> { + + private final AtomicLong value = new AtomicLong(); + + public MockLongValue(long value) { + this.value.set(value); + } + + @Override + public boolean succeeded() { + return true; + } + + public void setValue(long value) { + this.value.set(value); + } + + @Override + public Long preValue() { + return value.get(); + } + + @Override + public Long postValue() { + return value.get(); + } + + @Override + public AtomicStats getStats() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + } + + private class MockDirectoryCache implements Curator.DirectoryCache { + + /** The path this is caching and listening to */ + private final Path path; + + public MockDirectoryCache(Path path) { + this.path = path; + } + + @Override + public void start() {} + + @Override + public void addListener(PathChildrenCacheListener listener) { + listeners.add(path, listener); + } + + @Override + public List<ChildData> getCurrentData() { + List<ChildData> childData = new ArrayList<>(); + for (String childName : getChildren(path)) { + Path childPath = path.append(childName); + childData.add(new ChildData(childPath.getAbsolute(), null, getData(childPath).get())); + } + return childData; + } + + @Override + public ChildData getCurrentData(Path fullPath) { + if (!fullPath.getParentPath().equals(path)) { + throw new IllegalArgumentException("Path '" + fullPath + "' is not a child path of '" + path + "'"); + } + + return getData(fullPath).map(bytes -> new ChildData(fullPath.getAbsolute(), null, bytes)).orElse(null); + } + + @Override + public void close() {} + + private List<String> getChildren(Path path) { + try { + return MockCuratorFramework.this.getChildren().forPath(path.getAbsolute()); + } catch (KeeperException.NoNodeException e) { + return List.of(); + } catch (Exception e) { + throw new RuntimeException("Could not get children of " + path.getAbsolute(), e); + } + } + + private Optional<byte[]> getData(Path path) { + try { + return Optional.of(MockCuratorFramework.this.getData().forPath(path.getAbsolute())); + } + catch (KeeperException.NoNodeException e) { + return Optional.empty(); + } + catch (Exception e) { + throw new RuntimeException("Could not get data at " + path.getAbsolute(), e); + } + } + + } + + private class MockFileCache implements Curator.FileCache { + + /** The path this is caching and listening to */ + private final Path path; + + public MockFileCache(Path path) { + this.path = path; + } + + @Override + public void start() {} + + @Override + public void addListener(NodeCacheListener listener) { + listeners.add(path, listener); + } + + @Override + public ChildData getCurrentData() { + MemoryFileSystem.Node node = fileSystem.root().getNode(Paths.get(path.toString()), false); + if (node == null) return null; + return new ChildData("/" + path.toString(), null, node.getContent()); + } + + @Override + public void close() {} + + } + + // ----- The rest of this file is adapting the Curator (non-recipe) API to the ----- + // ----- file system methods above. ----- + // ----- There's nothing to see unless you are interested in an illustration of ----- + // ----- the folly of fluent API's or, more generally, mankind. ----- + + private abstract static class MockBackgroundACLPathAndBytesableBuilder<T> implements PathAndBytesable<T>, ProtectACLCreateModePathAndBytesable<T> { + + public BackgroundPathAndBytesable<T> withACL(List<ACL> list) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public ACLBackgroundPathAndBytesable<T> withMode(CreateMode createMode) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ACLCreateModeBackgroundPathAndBytesable<String> withProtection() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public T forPath(String s, byte[] bytes) throws Exception { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public T forPath(String s) throws Exception { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + } + + private class MockCreateBuilder extends MockBackgroundACLPathAndBytesableBuilder<String> implements CreateBuilder { + + private boolean createParents = false; + private CreateMode createMode = CreateMode.PERSISTENT; + + @Override + public ProtectACLCreateModePathAndBytesable<String> creatingParentsIfNeeded() { + createParents = true; + return this; + } + + @Override + public ACLCreateModeBackgroundPathAndBytesable<String> withProtection() { + // Protection against the server crashing after creating the file but before returning to the client. + // Not relevant for an in-memory mock, obviously + return this; + } + + public ACLBackgroundPathAndBytesable<String> withMode(CreateMode createMode) { + this.createMode = createMode; + return this; + } + + @Override + public CreateBackgroundModeACLable compressed() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ProtectACLCreateModePathAndBytesable<String> creatingParentContainersIfNeeded() { + // TODO: Add proper support for container nodes, see https://issues.apache.org/jira/browse/ZOOKEEPER-2163. + return creatingParentsIfNeeded(); + } + + @Override + @Deprecated + public ACLPathAndBytesable<String> withProtectedEphemeralSequential() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public String forPath(String s) throws Exception { + return createNode(s, new byte[0], createParents, createMode, fileSystem.root(), listeners); + } + + public String forPath(String s, byte[] bytes) throws Exception { + return createNode(s, bytes, createParents, createMode, fileSystem.root(), listeners); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<String> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + } + + private static class MockBackgroundPathableBuilder<T> implements BackgroundPathable<T>, Watchable<BackgroundPathable<T>> { + + @Override + public ErrorListenerPathable<T> inBackground() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<T> inBackground(Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathable<T> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public T forPath(String s) throws Exception { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public BackgroundPathable<T> watched() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public BackgroundPathable<T> usingWatcher(Watcher watcher) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public BackgroundPathable<T> usingWatcher(CuratorWatcher curatorWatcher) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + } + + private class MockGetChildrenBuilder extends MockBackgroundPathableBuilder<List<String>> implements GetChildrenBuilder { + + @Override + public WatchPathable<List<String>> storingStatIn(Stat stat) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public List<String> forPath(String path) throws Exception { + return getChildren(path, fileSystem.root()); + } + + } + + private class MockExistsBuilder extends MockBackgroundPathableBuilder<Stat> implements ExistsBuilder { + + @Override + public Stat forPath(String path) { + try { + MemoryFileSystem.Node node = getNode(path, fileSystem.root()); + Stat stat = new Stat(); + stat.setVersion(node.version()); + return stat; + } + catch (KeeperException.NoNodeException e) { + return null; + } + } + + @Override + public ExistsBuilderMain creatingParentContainersIfNeeded() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + } + + private class MockDeleteBuilder extends MockBackgroundPathableBuilder<Void> implements DeleteBuilder { + + private boolean deleteChildren = false; + + @Override + public BackgroundVersionable deletingChildrenIfNeeded() { + deleteChildren = true; + return this; + } + + @Override + public ChildrenDeletable guaranteed() { + return this; + } + + @Override + public BackgroundPathable<Void> withVersion(int i) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public Void forPath(String pathString) throws Exception { + deleteNode(pathString, deleteChildren, fileSystem.root(), listeners); + return null; + } + + } + + private class MockGetDataBuilder extends MockBackgroundPathableBuilder<byte[]> implements GetDataBuilder { + + @Override + public GetDataWatchBackgroundStatable decompressed() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public WatchPathable<byte[]> storingStatIn(Stat stat) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + public byte[] forPath(String path) throws Exception { + return getData(path, fileSystem.root()); + } + + } + + private class MockSetDataBuilder extends MockBackgroundACLPathAndBytesableBuilder<Stat> implements SetDataBuilder { + + @Override + public SetDataBackgroundVersionable compressed() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public BackgroundPathAndBytesable<Stat> withVersion(int i) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public Stat forPath(String path, byte[] bytes) throws Exception { + setData(path, bytes, fileSystem.root(), listeners); + return null; + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground(Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Object o) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ErrorListenerPathAndBytesable<Stat> inBackground(BackgroundCallback backgroundCallback, Object o, Executor executor) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + } + + /** Allows addition of directoryListeners which are never called */ + private static class MockListenable<T> implements Listenable<T> { + + @Override + public void addListener(T t) { + } + + @Override + public void addListener(T t, Executor executor) { + } + + @Override + public void removeListener(T t) { + } + + } + + private class MockCuratorTransactionFinal implements CuratorTransactionFinal { + + /** The new directory root in which the transactional changes are made */ + private final MemoryFileSystem.Node newRoot; + + private boolean committed = false; + + private final MockCuratorTransactionFinal.DelayedListener delayedListener = new MockCuratorTransactionFinal.DelayedListener(); + + public MockCuratorTransactionFinal() { + newRoot = fileSystem.root().clone(); + } + + @Override + public Collection<CuratorTransactionResult> commit() { + fileSystem.replaceRoot(newRoot); + committed = true; + delayedListener.commit(); + return null; // TODO + } + + @Override + public TransactionCreateBuilder create() { + ensureNotCommitted(); + return new MockCuratorTransactionFinal.MockTransactionCreateBuilder(); + } + + @Override + public TransactionDeleteBuilder delete() { + ensureNotCommitted(); + return new MockCuratorTransactionFinal.MockTransactionDeleteBuilder(); + } + + @Override + public TransactionSetDataBuilder setData() { + ensureNotCommitted(); + return new MockCuratorTransactionFinal.MockTransactionSetDataBuilder(); + } + + @Override + public TransactionCheckBuilder check() { + ensureNotCommitted(); + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + private void ensureNotCommitted() { + if (committed) throw new IllegalStateException("transaction already committed"); + } + + private class MockTransactionCreateBuilder implements TransactionCreateBuilder { + + private CreateMode createMode = CreateMode.PERSISTENT; + + @Override + public PathAndBytesable<CuratorTransactionBridge> withACL(List<ACL> list) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ACLCreateModePathAndBytesable<CuratorTransactionBridge> compressed() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public ACLPathAndBytesable<CuratorTransactionBridge> withMode(CreateMode createMode) { + this.createMode = createMode; + return this; + } + + @Override + public CuratorTransactionBridge forPath(String s, byte[] bytes) throws Exception { + createNode(s, bytes, false, createMode, newRoot, delayedListener); + return new MockCuratorTransactionFinal.MockCuratorTransactionBridge(); + } + + @Override + public CuratorTransactionBridge forPath(String s) throws Exception { + createNode(s, new byte[0], false, createMode, newRoot, delayedListener); + return new MockCuratorTransactionFinal.MockCuratorTransactionBridge(); + } + + } + + private class MockTransactionDeleteBuilder implements TransactionDeleteBuilder { + + @Override + public Pathable<CuratorTransactionBridge> withVersion(int i) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public CuratorTransactionBridge forPath(String path) throws Exception { + deleteNode(path, false, newRoot, delayedListener); + return new MockCuratorTransactionFinal.MockCuratorTransactionBridge(); + } + + } + + private class MockTransactionSetDataBuilder implements TransactionSetDataBuilder { + + @Override + public VersionPathAndBytesable<CuratorTransactionBridge> compressed() { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public PathAndBytesable<CuratorTransactionBridge> withVersion(int i) { + throw new UnsupportedOperationException("Not implemented in MockCurator"); + } + + @Override + public CuratorTransactionBridge forPath(String s, byte[] bytes) throws Exception { + MockCuratorFramework.this.setData(s, bytes, newRoot, delayedListener); + return new MockCuratorTransactionFinal.MockCuratorTransactionBridge(); + } + + @Override + public CuratorTransactionBridge forPath(String s) throws Exception { + MockCuratorFramework.this.setData(s, new byte[0], newRoot, delayedListener); + return new MockCuratorTransactionFinal.MockCuratorTransactionBridge(); + } + + } + + private class MockCuratorTransactionBridge implements CuratorTransactionBridge { + + @Override + public CuratorTransactionFinal and() { + return MockCuratorTransactionFinal.this; + } + + } + + /** A class which collects listen events and forwards them to the regular directoryListeners on commit */ + private class DelayedListener extends Listeners { + + private final List<Pair<Path, PathChildrenCacheEvent>> events = new ArrayList<>(); + + @Override + public void notify(Path path, PathChildrenCacheEvent event) { + events.add(new Pair<>(path, event)); + } + + public void commit() { + for (Pair<Path, PathChildrenCacheEvent> event : events) + listeners.notify(event.getFirst(), event.getSecond()); + } + + } + + } + +} diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/ConnectionSpecTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/ConnectionSpecTest.java new file mode 100644 index 00000000000..a518d8df843 --- /dev/null +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/ConnectionSpecTest.java @@ -0,0 +1,74 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator; + +import com.yahoo.net.HostName; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class ConnectionSpecTest { + + @Test + public void create() { + HostName.setHostNameForTestingOnly("host2"); + Config config = new Config(List.of(new Config.Server("host1", 10001), + new Config.Server("host2", 10002), + new Config.Server("host3", 10003))); + + { + ConnectionSpec spec = ConnectionSpec.create(config.servers, Config.Server::hostname, Config.Server::port, false); + assertEquals("host1:10001,host2:10002,host3:10003", spec.local()); + assertEquals("host1:10001,host2:10002,host3:10003", spec.ensemble()); + assertEquals(3, spec.ensembleSize()); + } + + { + ConnectionSpec specLocalAffinity = ConnectionSpec.create(config.servers, Config.Server::hostname, Config.Server::port, true); + assertEquals("host2:10002", specLocalAffinity.local()); + assertEquals("host1:10001,host2:10002,host3:10003", specLocalAffinity.ensemble()); + assertEquals(3, specLocalAffinity.ensembleSize()); + } + + { + ConnectionSpec specFromString = ConnectionSpec.create("host1:10001", "host1:10001,host2:10002"); + assertEquals("host1:10001", specFromString.local()); + assertEquals("host1:10001,host2:10002", specFromString.ensemble()); + assertEquals(2, specFromString.ensembleSize()); + } + } + + private static class Config { + + private final List<Server> servers; + + public Config(List<Server> servers) { + this.servers = servers; + } + + private static class Server { + + private final String hostname; + private final int port; + + public Server(String hostname, int port) { + this.hostname = hostname; + this.port = port; + } + + public String hostname() { + return hostname; + } + + public int port() { + return port; + } + } + + } + +} diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java index 2bf40c4e2bb..5341efaefe5 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/CuratorTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator; -import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.CuratorConfig; import com.yahoo.net.HostName; import org.apache.curator.test.TestingServer; import org.junit.After; @@ -61,45 +61,33 @@ public class CuratorTest { @Test public void require_that_server_count_is_correct() { - ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); - builder.zookeeperserver(createZKBuilder(localhost, port1)); - try (Curator curator = createCurator(new ConfigserverConfig(builder))) { + CuratorConfig.Builder builder = new CuratorConfig.Builder(); + builder.server(createZKBuilder(localhost, port1)); + try (Curator curator = createCurator(new CuratorConfig(builder))) { assertEquals(1, curator.zooKeeperEnsembleCount()); } } - @Test - public void localhost_affinity() { - String localhostHostName = "myhost"; - int localhostPort = 123; - - ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); - builder.zookeeperserver(createZKBuilder(localhostHostName, localhostPort)); - builder.zookeeperserver(createZKBuilder("otherhost", 345)); - ConfigserverConfig config = new ConfigserverConfig(builder); - - HostName.setHostNameForTestingOnly(localhostHostName); - - String localhostSpec = localhostHostName + ":" + localhostPort; - assertEquals(localhostSpec, Curator.createConnectionSpecForLocalhost(config)); - } - - private ConfigserverConfig createTestConfig() { - ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); - builder.zookeeperserver(createZKBuilder(localhost, port1)); - builder.zookeeperserver(createZKBuilder(localhost, port2)); - return new ConfigserverConfig(builder); + private CuratorConfig createTestConfig() { + CuratorConfig.Builder builder = new CuratorConfig.Builder(); + builder.server(createZKBuilder(localhost, port1)); + builder.server(createZKBuilder(localhost, port2)); + return new CuratorConfig(builder); } - private ConfigserverConfig.Zookeeperserver.Builder createZKBuilder(String hostname, int port) { - ConfigserverConfig.Zookeeperserver.Builder zkBuilder = new ConfigserverConfig.Zookeeperserver.Builder(); + private CuratorConfig.Server.Builder createZKBuilder(String hostname, int port) { + CuratorConfig.Server.Builder zkBuilder = new CuratorConfig.Server.Builder(); zkBuilder.hostname(hostname); zkBuilder.port(port); return zkBuilder; } - private Curator createCurator(ConfigserverConfig configserverConfig) { - return new Curator(configserverConfig, Optional.empty()); + private Curator createCurator(CuratorConfig curatorConfig) { + return new Curator(ConnectionSpec.create(curatorConfig.server(), + CuratorConfig.Server::hostname, + CuratorConfig.Server::port, + curatorConfig.zookeeperLocalhostAffinity()), + Optional.empty()); } private static class PortAllocator { |