From 32cdca292aa2832a06292a79104efecfe4ffdd5d Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Wed, 22 Apr 2020 11:29:14 +0200 Subject: Rename methods --- .../deployment/InternalStepRunnerTest.java | 4 +-- .../controller/integration/ConfigServerMock.java | 40 +++++++++++----------- .../controller/integration/NodeRepositoryMock.java | 34 ++++++++++-------- .../maintenance/CloudEventReporterTest.java | 8 ++--- .../maintenance/CostReportMaintainerTest.java | 2 +- .../controller/maintenance/OsUpgraderTest.java | 4 +-- .../maintenance/ResourceMeterMaintainerTest.java | 8 ++--- .../maintenance/ResourceTagMaintainerTest.java | 6 ++-- .../controller/maintenance/SystemUpgraderTest.java | 4 +-- .../hosted/controller/restapi/os/OsApiTest.java | 2 +- .../controller/versions/VersionStatusTest.java | 4 +-- 11 files changed, 61 insertions(+), 55 deletions(-) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 99325ff909d..d86d2960c74 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -212,8 +212,8 @@ public class InternalStepRunnerTest { Node systemTestNode = tester.configServer().nodeRepository().list(JobType.systemTest.zone(system()), app.instanceId()).iterator().next(); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().minus(Duration.ofSeconds(1))); - tester.configServer().nodeRepository().putByHostname(JobType.systemTest.zone(system()), - new Node.Builder(systemTestNode) + tester.configServer().nodeRepository().putNodes(JobType.systemTest.zone(system()), + new Node.Builder(systemTestNode) .serviceState(Node.ServiceState.allowedDown) .suspendedSince(tester.clock().instant()) .build()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 9e9d27fd744..d5a71d5adca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -100,23 +100,23 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer public void provision(ZoneId zone, ApplicationId application) { Node parent = nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream().findAny() .orElseThrow(() -> new IllegalStateException("No parent hosts in " + zone)); - nodeRepository().putByHostname(zone, new Node.Builder().hostname(hostFor(application, zone)) - .state(Node.State.reserved) - .type(NodeType.tenant) - .owner(application) - .parentHostname(parent.hostname()) - .currentVersion(initialVersion) - .wantedVersion(initialVersion) - .currentDockerImage(initialDockerImage) - .wantedDockerImage(initialDockerImage) - .currentOsVersion(Version.emptyVersion) - .wantedOsVersion(Version.emptyVersion) - .resources(new NodeResources(2, 8, 50, 1, slow, remote)) - .serviceState(Node.ServiceState.unorchestrated) - .flavor("d-2-8-50") - .clusterId("cluster") - .clusterType(Node.ClusterType.container) - .build()); + nodeRepository().putNodes(zone, new Node.Builder().hostname(hostFor(application, zone)) + .state(Node.State.reserved) + .type(NodeType.tenant) + .owner(application) + .parentHostname(parent.hostname()) + .currentVersion(initialVersion) + .wantedVersion(initialVersion) + .currentDockerImage(initialDockerImage) + .wantedDockerImage(initialDockerImage) + .currentOsVersion(Version.emptyVersion) + .wantedOsVersion(Version.emptyVersion) + .resources(new NodeResources(2, 8, 50, 1, slow, remote)) + .serviceState(Node.ServiceState.unorchestrated) + .flavor("d-2-8-50") + .clusterId("cluster") + .clusterType(Node.ClusterType.container) + .build()); } public HostName hostFor(ApplicationId application, ZoneId zone) { @@ -146,7 +146,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer .currentOsVersion(Version.emptyVersion).wantedOsVersion(Version.emptyVersion) .build()) .collect(Collectors.toList()); - nodeRepository().putByHostname(zone, nodes); + nodeRepository().putNodes(zone, nodes); convergeServices(application.id(), zone); } } @@ -206,7 +206,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } else { newNode = new Node.Builder(node).currentVersion(version).wantedVersion(version).build(); } - nodeRepository().putByHostname(zone, newNode); + nodeRepository().putNodes(zone, newNode); if (++n == nodeCount) break; } } @@ -355,7 +355,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer application.activate(); List nodes = nodeRepository.list(id.zoneId(), id.applicationId()); for (Node node : nodes) { - nodeRepository.putByHostname(id.zoneId(), new Node.Builder(node) + nodeRepository.putNodes(id.zoneId(), new Node.Builder(node) .state(Node.State.active) .wantedVersion(application.version().get()) .build()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 4aab21a44fe..1e535e7e9de 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -36,24 +36,33 @@ public class NodeRepositoryMock implements NodeRepository { private final Map> nodeRepository = new HashMap<>(); private final Map targetVersions = new HashMap<>(); - public void putByHostname(ZoneId zone, List nodes) { + /** Add or update given nodes in zone */ + public void putNodes(ZoneId zone, List nodes) { nodeRepository.putIfAbsent(zone, new HashMap<>()); nodeRepository.get(zone).putAll(nodes.stream().collect(Collectors.toMap(Node::hostname, Function.identity()))); } - public void putByHostname(ZoneId zone, Node node) { - putByHostname(zone, Collections.singletonList(node)); + /** Add or update given node in zone */ + public void putNodes(ZoneId zone, Node node) { + putNodes(zone, Collections.singletonList(node)); } - public void removeByHostname(ZoneId zone, List nodes) { + /** Remove given nodes from zone */ + public void removeNodes(ZoneId zone, List nodes) { nodes.forEach(node -> nodeRepository.get(zone).remove(node.hostname())); } + /** Remove all nodes in all zones */ public void clear() { nodeRepository.clear(); } + /** Replace nodes in zone with given nodes */ + public void setNodes(ZoneId zone, List nodes) { + nodeRepository.put(zone, nodes.stream().collect(Collectors.toMap(Node::hostname, Function.identity()))); + } + public Node require(HostName hostName) { return nodeRepository.values().stream() .map(zoneNodes -> zoneNodes.get(hostName)) @@ -62,11 +71,8 @@ public class NodeRepositoryMock implements NodeRepository { .orElseThrow(() -> new NoSuchElementException("No node with the hostname " + hostName + " is known.")); } - public void addNodes(ZoneId zone, List nodes) { - nodeRepository.put(zone, nodes.stream().collect(Collectors.toMap(Node::hostname, Function.identity()))); - } - - public void addFixedNodes(ZoneId zone) { + /** Replace nodes in zone with a fixed set of nodes */ + public void setFixedNodes(ZoneId zone) { var nodeA = new Node.Builder() .hostname(HostName.from("hostA")) .parentHostname(HostName.from("parentHostA")) @@ -98,7 +104,7 @@ public class NodeRepositoryMock implements NodeRepository { .clusterId("clusterB") .clusterType(Node.ClusterType.container) .build(); - addNodes(zone, List.of(nodeA, nodeB)); + setNodes(zone, List.of(nodeA, nodeB)); } @Override @@ -168,7 +174,7 @@ public class NodeRepositoryMock implements NodeRepository { .stream() .filter(node -> node.type() == type) .map(node -> new Node.Builder(node).wantedVersion(version).build()) - .forEach(node -> putByHostname(zone, node)); + .forEach(node -> putNodes(zone, node)); } @Override @@ -184,7 +190,7 @@ public class NodeRepositoryMock implements NodeRepository { .stream() .filter(node -> node.type() == type) .map(node -> new Node.Builder(node).wantedOsVersion(version).build()) - .forEach(node -> putByHostname(zone, node)); + .forEach(node -> putNodes(zone, node)); } @Override @@ -219,8 +225,8 @@ public class NodeRepositoryMock implements NodeRepository { List nodes = hostname.map(this::require) .map(Collections::singletonList) .orElse(list(deployment.zoneId(), deployment.applicationId())); - putByHostname(deployment.zoneId(), - nodes.stream().map(modification).collect(Collectors.toList())); + putNodes(deployment.zoneId(), + nodes.stream().map(modification).collect(Collectors.toList())); } public void requestRestart(DeploymentId deployment, Optional hostname) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java index 1c66a01db8e..369ecaf5eb7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java @@ -97,7 +97,7 @@ public class CloudEventReporterTest { awsZone1, awsZone2); - tester.configServer().nodeRepository().putByHostname( + tester.configServer().nodeRepository().putNodes( nonAwsZone.getId(), createNodesWithHostnames( "host1.com", @@ -105,7 +105,7 @@ public class CloudEventReporterTest { "host3.com" ) ); - tester.configServer().nodeRepository().putByHostname( + tester.configServer().nodeRepository().putNodes( awsZone1.getId(), createNodesWithHostnames( "host1.com", @@ -113,14 +113,14 @@ public class CloudEventReporterTest { "host3.com" ) ); - tester.configServer().nodeRepository().putByHostname( + tester.configServer().nodeRepository().putNodes( awsZone2.getId(), createNodesWithHostnames( "host4.com", "host5.com" ) ); - tester.configServer().nodeRepository().putByHostname( + tester.configServer().nodeRepository().putNodes( awsZone2.getId(), List.of(createNode("confighost.com", NodeType.confighost)) ); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index ca624281e94..b19003ff734 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -54,7 +54,7 @@ public class CostReportMaintainerTest { private void addNodes() { for (var zone : tester.zoneRegistry().zones().all().zones()) { - tester.configServer().nodeRepository().addFixedNodes(zone.getId()); + tester.configServer().nodeRepository().setFixedNodes(zone.getId()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 9d910afd476..baa4b29ac1d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -146,7 +146,7 @@ public class OsUpgraderTest { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putByHostname(zone, new Node.Builder(node).state(Node.State.failed).build()); + nodeRepository().putNodes(zone, new Node.Builder(node).state(Node.State.failed).build()); } /** Simulate OS upgrade of nodes allocated to application. In a real system this is done by the node itself */ @@ -154,7 +154,7 @@ public class OsUpgraderTest { assertWanted(version, application, zones); for (ZoneId zone : zones) { for (Node node : nodesRequiredToUpgrade(zone, application)) { - nodeRepository().putByHostname(zone, new Node.Builder(node).wantedOsVersion(version).currentOsVersion(version).build()); + nodeRepository().putNodes(zone, new Node.Builder(node).wantedOsVersion(version).currentOsVersion(version).build()); } assertCurrent(version, application, zone); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index e0a97d32597..8379722be66 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -64,10 +64,10 @@ public class ResourceMeterMaintainerTest { nonAwsZone, awsZone1, awsZone2); - tester.configServer().nodeRepository().addFixedNodes(nonAwsZone.getId()); - tester.configServer().nodeRepository().addFixedNodes(awsZone1.getId()); - tester.configServer().nodeRepository().addFixedNodes(awsZone2.getId()); - tester.configServer().nodeRepository().putByHostname( + tester.configServer().nodeRepository().setFixedNodes(nonAwsZone.getId()); + tester.configServer().nodeRepository().setFixedNodes(awsZone1.getId()); + tester.configServer().nodeRepository().setFixedNodes(awsZone2.getId()); + tester.configServer().nodeRepository().putNodes( awsZone1.getId(), createNodesInState( Node.State.provisioned, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index 653a4c5394b..66b125186e0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -46,9 +46,9 @@ public class ResourceTagMaintainerTest { nonAwsZone, awsZone1, awsZone2); - tester.configServer().nodeRepository().addFixedNodes(nonAwsZone.getId()); - tester.configServer().nodeRepository().addFixedNodes(awsZone1.getId()); - tester.configServer().nodeRepository().addFixedNodes(awsZone2.getId()); + tester.configServer().nodeRepository().setFixedNodes(nonAwsZone.getId()); + tester.configServer().nodeRepository().setFixedNodes(awsZone1.getId()); + tester.configServer().nodeRepository().setFixedNodes(awsZone2.getId()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 140c37cdd94..f1031c90344 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -302,7 +302,7 @@ public class SystemUpgraderTest { assertWantedVersion(application, version, first, rest); Stream.concat(Stream.of(first), Stream.of(rest)).forEach(zone -> { for (Node node : listNodes(zone, application)) { - nodeRepository().putByHostname( + nodeRepository().putNodes( zone.getId(), new Node.Builder(node).currentVersion(node.wantedVersion()).build()); } @@ -326,7 +326,7 @@ public class SystemUpgraderTest { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putByHostname( + nodeRepository().putNodes( zone.getId(), new Node.Builder(node).state(Node.State.failed).build()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 7370cf12d5a..9f63c714786 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -144,7 +144,7 @@ public class OsApiTest extends ControllerContainerTest { var targetVersion = nodeRepository().targetVersionsOf(zone).osVersion(application.nodeType()); for (Node node : nodeRepository().list(zone, application.id())) { var version = targetVersion.orElse(node.wantedOsVersion()); - nodeRepository().putByHostname(zone, new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build()); + nodeRepository().putNodes(zone, new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build()); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index f60d11693d8..44e172d9f93 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -72,7 +72,7 @@ public class VersionStatusTest { for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) { for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { Node upgradedNode = new Node.Builder(node).currentVersion(version1).build(); - tester.configServer().nodeRepository().putByHostname(zone.getId(), upgradedNode); + tester.configServer().nodeRepository().putNodes(zone.getId(), upgradedNode); break; } } @@ -116,7 +116,7 @@ public class VersionStatusTest { for (ZoneApi zone : tester.controller().zoneRegistry().zones().all().zones()) { for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { Node downgradedNode = new Node.Builder(node).currentVersion(ancientVersion).build(); - tester.configServer().nodeRepository().putByHostname(zone.getId(), downgradedNode); + tester.configServer().nodeRepository().putNodes(zone.getId(), downgradedNode); break; } } -- cgit v1.2.3 From 5f33236d193dc39d985da93da2a753557ce28404 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 22 Apr 2020 12:36:22 +0200 Subject: Gc dead code --- .../java/com/yahoo/config/provision/Capacity.java | 48 ---------------------- .../com/yahoo/config/provision/Provisioner.java | 3 -- .../provisioning/NodeRepositoryProvisioner.java | 12 ------ .../provision/testutils/MockProvisioner.java | 6 --- 4 files changed, 69 deletions(-) diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 09ee25fb437..3c7a14b9496 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -40,37 +40,6 @@ public final class Capacity { ") must be divisible by the number of groups (" + resources.groups() + ")"); } - /** Returns the number of nodes requested */ - @Deprecated // TODO: Remove after April 2020 - public int nodeCount() { return min.nodes(); } - - /** Returns the number of nodes requested (across all groups), or 0 if not specified */ - @Deprecated // TODO: Remove after April 2020 - public int nodes() { return min.nodes(); } - - /** Returns the number of groups requested, or 0 if not specified */ - @Deprecated // TODO: Remove after April 2020 - public int groups() { return min.groups(); } - - /** - * The node flavor requested, or empty if no legacy flavor name has been used. - * This may be satisfied by the requested flavor or a suitable replacement. - * - * @deprecated use nodeResources instead - */ - @Deprecated // TODO: Remove after March 2020 - public Optional flavor() { - if (nodeResources().isEmpty()) return Optional.empty(); - return Optional.of(min.nodeResources().toString()); - } - - /** Returns the resources requested for each node, or empty to leave this decision to provisioning */ - @Deprecated // TODO: Remove after March 2020 - public Optional nodeResources() { - if (min.nodeResources() == NodeResources.unspecified) return Optional.empty(); - return Optional.of(min.nodeResources()); - } - public ClusterResources minResources() { return min; } public ClusterResources maxResources() { return max; } @@ -114,23 +83,6 @@ public final class Capacity { return new Capacity(min, max, required, canFail, NodeType.tenant); } - /** Create a non-required, failable capacity request */ - @Deprecated // TODO: Remove after April 2020 - public static Capacity fromCount(int nodes, NodeResources resources) { - return fromCount(nodes, resources, false, true); - } - - @Deprecated // TODO: Remove after April 2020 - public static Capacity fromCount(int nodes, NodeResources resources, boolean required, boolean canFail) { - return fromCount(nodes, Optional.of(resources), required, canFail); - } - - @Deprecated // TODO: Remove after April 2020 - public static Capacity fromCount(int nodes, Optional resources, boolean required, boolean canFail) { - return from(new ClusterResources(nodes, 0, resources.orElse(NodeResources.unspecified)), - required, canFail, NodeType.tenant); - } - /** Creates this from a node type */ public static Capacity fromRequiredNodeType(NodeType type) { return from(new ClusterResources(0, 0, NodeResources.unspecified), true, false, type); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java index e308d631442..2a1528f5368 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java @@ -24,9 +24,6 @@ public interface Provisioner { */ List prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger); - @Deprecated // TODO: Remove after April 2020 - List prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger); - /** * Activates the allocation of nodes to this application captured in the hosts argument. * diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 103543917bc..4d99cd3acb1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -73,18 +73,6 @@ public class NodeRepositoryProvisioner implements Provisioner { this.activator = new Activator(nodeRepository, loadBalancerProvisioner); } - - /** - * Returns a list of nodes in the prepared or active state, matching the given constraints. - * The nodes are ordered by increasing index number. - */ - @Deprecated // TODO: Remove after April 2020 - @Override - public List prepare(ApplicationId application, ClusterSpec cluster, Capacity requestedCapacity, - int wantedGroups, ProvisionLogger logger) { - return prepare(application, cluster, requestedCapacity.withGroups(wantedGroups), logger); - } - /** * Returns a list of nodes in the prepared or active state, matching the given constraints. * The nodes are ordered by increasing index number. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java index 045d0cad1ad..12731e30f46 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java @@ -19,12 +19,6 @@ import java.util.List; */ public class MockProvisioner implements Provisioner { - @Override - @Deprecated // TODO: Remove after April 2020 - public List prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - return Collections.emptyList(); - } - @Override public List prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { return Collections.emptyList(); -- cgit v1.2.3 From 10a34ffd6e29c0c396c56d315ef53f010a02a505 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 22 Apr 2020 13:17:06 +0200 Subject: Fix buffer overflow in vespalib datastore buffer type test. --- vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp index 6a51b9192ce..3b748d1d4c7 100644 --- a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp +++ b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp @@ -36,19 +36,19 @@ struct Fixture { Setup setup; IntBufferType bufferType; size_t deadElems; - int buffer; + int buffer[ARRAYS_SIZE]; Fixture(const Setup &setup_) : setup(setup_), bufferType(ARRAYS_SIZE, setup._minArrays, MAX_ARRAYS, NUM_ARRAYS_FOR_NEW_BUFFER, setup._allocGrowFactor), deadElems(0), - buffer(0) + buffer() {} ~Fixture() { bufferType.onHold(&setup._usedElems); bufferType.onFree(setup._usedElems); } void onActive() { - bufferType.onActive(setup._bufferId, &setup._usedElems, deadElems, &buffer); + bufferType.onActive(setup._bufferId, &setup._usedElems, deadElems, &buffer[0]); } size_t arraysToAlloc() { return bufferType.calcArraysToAlloc(setup._bufferId, setup._neededElems, setup._resizing); -- cgit v1.2.3 From 435783bc79fca6d2c8fd3e03ac0ce12ab0e23c28 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 22 Apr 2020 13:19:45 +0200 Subject: Gc dead code --- .../java/com/yahoo/vespa/config/server/deploy/DeployTester.java | 6 ------ .../java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java | 6 ------ .../com/yahoo/vespa/config/server/session/SessionPreparerTest.java | 6 ------ 3 files changed, 18 deletions(-) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 84987bce32e..06db7018727 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -298,12 +298,6 @@ public class DeployTester { this.hostProvisioner = hostProvisioner; } - @Override - @Deprecated // TODO: Remove after April 2020 - public List prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - return hostProvisioner.prepare(cluster, capacity.withGroups(groups), logger); - } - @Override public List prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { return hostProvisioner.prepare(cluster, capacity, logger); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index 5b0bb7885d8..fa0bbac05c8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -236,12 +236,6 @@ public class SessionHandlerTest { public ApplicationId lastApplicationId; public Collection lastHosts; - @Override - @Deprecated // TODO: Remove after April 2020 - public List prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - throw new UnsupportedOperationException(); - } - @Override public List prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { throw new UnsupportedOperationException(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index a3b0f3ec44a..66891e54fb9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -341,12 +341,6 @@ public class SessionPreparerTest { private static class FailWithTransientExceptionProvisioner implements Provisioner { - @Override - @Deprecated // TODO: Remove after April 2020 - public List prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception")); - } - @Override public List prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception")); -- cgit v1.2.3 From e3e2f9a5b467498758cf505bfb0ef5d7e696d200 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 22 Apr 2020 13:21:49 +0200 Subject: Fix reference leak in fnet echo client example program. --- fnet/src/examples/frt/rpc/echo_client.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/fnet/src/examples/frt/rpc/echo_client.cpp b/fnet/src/examples/frt/rpc/echo_client.cpp index cc406224135..06f4ef0ee5b 100644 --- a/fnet/src/examples/frt/rpc/echo_client.cpp +++ b/fnet/src/examples/frt/rpc/echo_client.cpp @@ -84,6 +84,7 @@ EchoClient::Main() printf("Return values != parameters.\n"); } req->SubRef(); + target->SubRef(); return 0; } -- cgit v1.2.3 From b75c5b10b6805c8908f365d3bb173ad3086fc306 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 22 Apr 2020 13:36:29 +0200 Subject: Fix reference leak in jrt echo client test program. --- jrt_test/src/tests/echo/echo-client.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/jrt_test/src/tests/echo/echo-client.cpp b/jrt_test/src/tests/echo/echo-client.cpp index c1ffe29e1ce..3a87e38e6e0 100644 --- a/jrt_test/src/tests/echo/echo-client.cpp +++ b/jrt_test/src/tests/echo/echo-client.cpp @@ -77,6 +77,7 @@ public: printf("Return values != parameters.\n"); } req->SubRef(); + target->SubRef(); return 0; } }; -- cgit v1.2.3 From c8dde657c38721f3f55182143a892e40bdb14db3 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 21 Apr 2020 11:08:56 +0200 Subject: Use ClusterType in NodeMembership --- .../noderepository/NodeMembership.java | 48 ++++++++++++++++++++-- .../node/admin/docker/DockerOperationsImpl.java | 2 +- .../maintenance/coredump/CoredumpHandler.java | 6 +-- .../node/admin/task/util/file/FileFinder.java | 2 +- 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java index 22633f67463..3d89b3a991a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java @@ -1,28 +1,35 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; +import java.util.Objects; + /** * @author freva */ public class NodeMembership { - private final String clusterType; + private final ClusterType clusterType; private final String clusterId; private final String group; private final int index; private final boolean retired; public NodeMembership(String clusterType, String clusterId, String group, int index, boolean retired) { - this.clusterType = clusterType; + this.clusterType = new ClusterType(clusterType); this.clusterId = clusterId; this.group = group; this.index = index; this.retired = retired; } - public String clusterType() { + public ClusterType type() { return clusterType; } + /** DEPRECATED: Use {@link #type()} instead */ + public String clusterType() { + return clusterType.value(); + } + public String clusterId() { return clusterId; } @@ -74,4 +81,39 @@ public class NodeMembership { " retired = " + retired + " }"; } + + public static class ClusterType { + private final String type; + + private ClusterType(String type) { + this.type = Objects.requireNonNull(type); + } + + public boolean isAdmin() { return "admin".equals(type); } + public boolean isContent() { return "content".equals(type) || "combined".equals(type); } + public boolean isContainer() { return "container".equals(type) || "combined".equals(type); } + + public String value() { + return type; + } + + @Override + public String toString() { + return type; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ClusterType that = (ClusterType) o; + return type.equals(that.type); + } + + @Override + public int hashCode() { + return type.hashCode(); + } + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index f5bb46ef62a..d7f248250a8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -88,7 +88,7 @@ public class DockerOperationsImpl implements DockerOperations { if (context.nodeType() != NodeType.proxy && context.nodeType() != NodeType.controller) command.withSecurityOpt("no-new-privileges"); - if (context.node().membership().map(NodeMembership::clusterType).map("content"::equalsIgnoreCase).orElse(false)) + if (context.node().membership().map(m -> m.type().isContent()).orElse(false)) command.withSecurityOpt("seccomp=unconfined"); DockerNetworking networking = context.dockerNetworking(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 6aa5a8d9fc1..9d20d973676 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; @@ -44,7 +42,7 @@ public class CoredumpHandler { private static final String LZ4_PATH = "/usr/bin/lz4"; private static final String PROCESSING_DIRECTORY_NAME = "processing"; private static final String METADATA_FILE_NAME = "metadata.json"; - private static final String COREDUMP_FILENAME_PREFIX = "dump_"; + public static final String COREDUMP_FILENAME_PREFIX = "dump_"; private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName()); private final ObjectMapper objectMapper = new ObjectMapper(); @@ -245,7 +243,7 @@ public class CoredumpHandler { node.membership().ifPresent(membership -> dimensionsBuilder - .add("clustertype", membership.clusterType()) + .add("clustertype", membership.type().value()) .add("clusterid", membership.clusterId()) ); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java index f10ebd5e315..f0b064373d2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java @@ -63,7 +63,7 @@ public class FileFinder { /** * Predicate that will be used to match files and directories under the base path. * - * NOTE: Consequtive calls to this method are ANDed (this include the initial filter from + * NOTE: Consecutive calls to this method are ANDed (this include the initial filter from * {@link #files(Path)} or {@link #directories(Path)}. */ public FileFinder match(Predicate matcher) { -- cgit v1.2.3 From 95a6fc2366a2348f378ffc1c251296fc95a80726 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 21 Apr 2020 11:09:42 +0200 Subject: Throw ConvergenceException instead --- .../vespa/hosted/node/admin/maintenance/StorageMaintainer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 1684c0dd2a1..c1897c3c416 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -89,7 +89,7 @@ public class StorageMaintainer { String[] results = output.split("\t"); if (results.length != 2) { - throw new RuntimeException("Result from disk usage command not as expected: " + output); + throw new ConvergenceException("Result from disk usage command not as expected: " + output); } return 1024 * Long.parseLong(results[0]); @@ -174,11 +174,11 @@ public class StorageMaintainer { String output = uncheck(() -> Files.readAllLines(Paths.get("/proc/cpuinfo")).stream() .filter(line -> line.startsWith("microcode")) .findFirst() - .orElseThrow(() -> new RuntimeException("No microcode information found in /proc/cpuinfo"))); + .orElseThrow(() -> new ConvergenceException("No microcode information found in /proc/cpuinfo"))); String[] results = output.split(":"); if (results.length != 2) { - throw new RuntimeException("Result from detect microcode command not as expected: " + output); + throw new ConvergenceException("Result from detect microcode command not as expected: " + output); } return results[1].trim(); -- cgit v1.2.3 From a16e6385b4a4ff240435b7b3acc9c3677cdd551c Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 22 Apr 2020 14:17:25 +0200 Subject: Revert "Dump stack trace several times if config proxy does not start" --- config-proxy/src/main/sh/vespa-config-ctl.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index 91e4cd39bd2..c6c1afea62c 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -153,11 +153,6 @@ case $1 in elif ! vespa-ping-configproxy -s $hname then echo "failed to ping config proxy $hname" 1>&2 - # TODO: Dump stack trace for debugging, remove after April 2020 - kill -3 `pgrep -f -n configproxy` - sleep 2 - kill -3 `pgrep -f -n configproxy` - sleep 2 kill -3 `pgrep -f -n configproxy` fi -- cgit v1.2.3 From b79f094fd1327d6e124afb9367d8352076216e37 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Wed, 22 Apr 2020 11:48:21 +0200 Subject: Measure platform and OS change duration per host --- .../integration/configserver/NodeRepository.java | 2 +- .../controller/maintenance/MetricsReporter.java | 83 +++++---- .../persistence/NodeVersionSerializer.java | 11 +- .../hosted/controller/versions/NodeVersion.java | 46 +++-- .../hosted/controller/versions/NodeVersions.java | 51 +++--- .../controller/versions/OsVersionStatus.java | 35 ++-- .../hosted/controller/versions/VersionStatus.java | 24 +-- .../vespa/hosted/controller/ControllerTester.java | 2 +- .../deployment/InternalStepRunnerTest.java | 6 +- .../controller/integration/ConfigServerMock.java | 26 ++- .../maintenance/MetricsReporterTest.java | 185 +++++++++++++++------ .../persistence/OsVersionStatusSerializerTest.java | 15 +- .../persistence/VersionStatusSerializerTest.java | 14 +- .../restapi/deployment/DeploymentApiTest.java | 6 +- 14 files changed, 292 insertions(+), 214 deletions(-) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 05e9707473c..c5a016d53c8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -49,7 +49,7 @@ public interface NodeRepository { .collect(Collectors.toUnmodifiableList()); } - /** List all nodes in zone owned by given application */ + /** List all nodes in zone having given hostnames */ default List list(ZoneId zone, List hostnames) { return listNodes(zone, hostnames).nodes().stream() .map(NodeRepository::toNode) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 9c414ce8348..d70c65f343a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import com.yahoo.vespa.hosted.controller.deployment.JobList; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -20,6 +21,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -38,18 +40,21 @@ public class MetricsReporter extends Maintainer { public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades"; public static final String DEPLOYMENT_BUILD_AGE_SECONDS = "deployment.buildAgeSeconds"; public static final String DEPLOYMENT_WARNINGS = "deployment.warnings"; - public static final String NODES_FAILING_SYSTEM_UPGRADE = "deployment.nodesFailingSystemUpgrade"; + // TODO(mpolden): Remove these two metrics + public static final String NODES_FAILING_PLATFORM_UPGRADE = "deployment.nodesFailingSystemUpgrade"; public static final String NODES_FAILING_OS_UPGRADE = "deployment.nodesFailingOsUpgrade"; + public static final String OS_CHANGE_DURATION = "deployment.osChangeDuration"; + public static final String PLATFORM_CHANGE_DURATION = "deployment.platformChangeDuration"; public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; - // The time a node belonging to a system application can spend from being told to upgrade until the upgrade is - // completed. Nodes exceeding this time are counted as failures. - private static final Duration NODE_UPGRADE_TIMEOUT = Duration.ofMinutes(90); + // The time a system application node can spend after suspending for Vespa upgrade until the upgrade is completed. + // Nodes exceeding this budget are counted as failures. + private static final Duration PLATFORM_UPGRADE_BUDGET = Duration.ofMinutes(30); - // The time a single node can spend performing an OS upgrade after being told to upgrade. Nodes exceeding this time - // multiplied by the number of nodes upgrading are counted as failures. - private static final Duration OS_UPGRADE_TIME_ALLOWANCE_PER_NODE = Duration.ofMinutes(30); + // The time a system application node can spend after suspending foor OS upgrade until the upgrade is completed. + // Nodes exceeding this budget are counted as failures. + private static final Duration OS_UPGRADE_BUDGET = Duration.ofMinutes(30); private final Metric metric; private final Clock clock; @@ -65,7 +70,7 @@ public class MetricsReporter extends Maintainer { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); - reportNodesFailingUpgrade(); + reportChangeDurations(); } private void reportRemainingRotations() { @@ -107,43 +112,44 @@ public class MetricsReporter extends Maintainer { metric.createContext(Map.of())); } - private void reportNodesFailingUpgrade() { - metric.set(NODES_FAILING_SYSTEM_UPGRADE, nodesFailingSystemUpgrade(), metric.createContext(Map.of())); - metric.set(NODES_FAILING_OS_UPGRADE, nodesFailingOsUpgrade(), metric.createContext(Map.of())); + private void reportChangeDurations() { + var platformChangeDurations = platformChangeDurations(); + var osChangeDurations = osChangeDurations(); + var nodesFailingSystemUpgrade = platformChangeDurations.values().stream() + .filter(duration -> duration.compareTo(PLATFORM_UPGRADE_BUDGET) > 0) + .count(); + var nodesFailingOsUpgrade = osChangeDurations.values().stream() + .filter(duration -> duration.compareTo(OS_UPGRADE_BUDGET) > 0) + .count(); + metric.set(NODES_FAILING_PLATFORM_UPGRADE, nodesFailingSystemUpgrade, metric.createContext(Map.of())); + metric.set(NODES_FAILING_OS_UPGRADE, nodesFailingOsUpgrade, metric.createContext(Map.of())); + platformChangeDurations.forEach((nodeVersion, duration) -> { + metric.set(PLATFORM_CHANGE_DURATION, duration.toSeconds(), metric.createContext(dimensions(nodeVersion))); + }); + osChangeDurations.forEach((nodeVersion, duration) -> { + metric.set(OS_CHANGE_DURATION, duration.toSeconds(), metric.createContext(dimensions(nodeVersion))); + }); } - private int nodesFailingSystemUpgrade() { - if (!controller().versionStatus().isUpgrading()) return 0; - return nodesFailingUpgrade(controller().versionStatus().versions(), (vespaVersion) -> { - if (vespaVersion.confidence() == VespaVersion.Confidence.broken) return NodeVersions.EMPTY; - return vespaVersion.nodeVersions(); - }, NODE_UPGRADE_TIMEOUT); + private Map platformChangeDurations() { + return changeDurations(controller().versionStatus().versions(), VespaVersion::nodeVersions); } - private int nodesFailingOsUpgrade() { - var allNodeVersions = controller().osVersionStatus().versions().values(); - var totalTimeout = 0L; - for (var nodeVersions : allNodeVersions) { - for (var nodeVersion : nodeVersions.asMap().values()) { - if (!nodeVersion.changing()) continue; - totalTimeout += OS_UPGRADE_TIME_ALLOWANCE_PER_NODE.toMillis(); - } - } - return nodesFailingUpgrade(allNodeVersions, Function.identity(), Duration.ofMillis(totalTimeout)); + private Map osChangeDurations() { + return changeDurations(controller().osVersionStatus().versions().values(), Function.identity()); } - private int nodesFailingUpgrade(Collection collection, Function nodeVersionsFunction, Duration timeout) { - var nodesFailingUpgrade = 0; - var acceptableInstant = clock.instant().minus(timeout); - for (var object : collection) { - for (var nodeVersion : nodeVersionsFunction.apply(object).asMap().values()) { - if (!nodeVersion.changing()) continue; - if (nodeVersion.changedAt().isBefore(acceptableInstant)) nodesFailingUpgrade++; + private Map changeDurations(Collection versions, Function versionsGetter) { + var now = clock.instant(); + var durations = new HashMap(); + for (var version : versions) { + for (var nodeVersion : versionsGetter.apply(version).asMap().values()) { + durations.put(nodeVersion, nodeVersion.changeDuration(now)); } } - return nodesFailingUpgrade; + return durations; } - + private static double deploymentFailRatio(DeploymentStatusList statuses) { return statuses.asList().stream() .mapToInt(status -> status.hasFailures() ? 1 : 0) @@ -198,6 +204,11 @@ public class MetricsReporter extends Maintainer { "app",application.application().value() + "." + application.instance().value()); } + private static Map dimensions(NodeVersion nodeVersion) { + return Map.of("host", nodeVersion.hostname().value(), + "zone", nodeVersion.zone().value()); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java index 6e9def40e44..f9f8de96591 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java @@ -11,8 +11,6 @@ import com.yahoo.slime.Inspector; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; -import java.time.Instant; - /** * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.NodeVersion}. * @@ -30,7 +28,7 @@ public class NodeVersionSerializer { private static final String hostnameField = "hostname"; private static final String zoneField = "zone"; private static final String wantedVersionField = "wantedVersion"; - private static final String changedAtField = "changedAt"; + private static final String suspendedAtField = "suspendedAt"; public void nodeVersionsToSlime(NodeVersions nodeVersions, Cursor array) { for (var nodeVersion : nodeVersions.asMap().values()) { @@ -38,7 +36,8 @@ public class NodeVersionSerializer { nodeVersionObject.setString(hostnameField, nodeVersion.hostname().value()); nodeVersionObject.setString(zoneField, nodeVersion.zone().value()); nodeVersionObject.setString(wantedVersionField, nodeVersion.wantedVersion().toFullString()); - nodeVersionObject.setLong(changedAtField, nodeVersion.changedAt().toEpochMilli()); + nodeVersion.suspendedAt().ifPresent(suspendedAt -> nodeVersionObject.setLong(suspendedAtField, + suspendedAt.toEpochMilli())); } } @@ -48,8 +47,8 @@ public class NodeVersionSerializer { var hostname = HostName.from(entry.field(hostnameField).asString()); var zone = ZoneId.from(entry.field(zoneField).asString()); var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString()); - var changedAt = Instant.ofEpochMilli(entry.field(changedAtField).asLong()); - nodeVersions.put(hostname, new NodeVersion(hostname, zone, version, wantedVersion, changedAt)); + var suspendedAt = Serializers.optionalInstant(entry.field(suspendedAtField)); + nodeVersions.put(hostname, new NodeVersion(hostname, zone, version, wantedVersion, suspendedAt)); }); return new NodeVersions(nodeVersions.build()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java index d82cddb4779..4a00add411f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java @@ -5,8 +5,10 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; +import java.time.Duration; import java.time.Instant; import java.util.Objects; +import java.util.Optional; /** * Version information for a node allocated to a {@link com.yahoo.vespa.hosted.controller.application.SystemApplication}. @@ -21,14 +23,15 @@ public class NodeVersion { private final ZoneId zone; private final Version currentVersion; private final Version wantedVersion; - private final Instant changedAt; + private final Optional suspendedAt; - public NodeVersion(HostName hostname, ZoneId zone, Version currentVersion, Version wantedVersion, Instant changedAt) { + public NodeVersion(HostName hostname, ZoneId zone, Version currentVersion, Version wantedVersion, + Optional suspendedAt) { this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); this.zone = Objects.requireNonNull(zone, "zone must be non-null"); this.currentVersion = Objects.requireNonNull(currentVersion, "version must be non-null"); this.wantedVersion = Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null"); - this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); + this.suspendedAt = Objects.requireNonNull(suspendedAt, "suspendedAt must be non-null"); } /** Hostname of this */ @@ -51,31 +54,39 @@ public class NodeVersion { return wantedVersion; } - /** Returns whether this is changing (upgrading or downgrading) */ - public boolean changing() { - return !currentVersion.equals(wantedVersion); + /** Returns the duration of the change in this, measured relative to instant */ + public Duration changeDuration(Instant instant) { + if (!changing()) return Duration.ZERO; + if (suspendedAt.isEmpty()) return Duration.ZERO; // Node hasn't suspended to apply the change yet + return Duration.between(suspendedAt.get(), instant).abs(); } - /** The most recent time the version of this changed */ - public Instant changedAt() { - return changedAt; + /** The most recent time the node referenced by this suspended. This is empty if the node is not suspended. */ + public Optional suspendedAt() { + return suspendedAt; } /** Returns a copy of this with current version set to given version */ - public NodeVersion withCurrentVersion(Version version, Instant changedAt) { + public NodeVersion withCurrentVersion(Version version) { if (currentVersion.equals(version)) return this; - return new NodeVersion(hostname, zone, version, wantedVersion, changedAt); + return new NodeVersion(hostname, zone, version, wantedVersion, suspendedAt); } /** Returns a copy of this with wanted version set to given version */ public NodeVersion withWantedVersion(Version version) { if (wantedVersion.equals(version)) return this; - return new NodeVersion(hostname, zone, currentVersion, version, changedAt); + return new NodeVersion(hostname, zone, currentVersion, version, suspendedAt); + } + + /** Returns a copy of this with wanted version set to given version */ + public NodeVersion withSuspendedAt(Optional suspendedAt) { + if (suspendedAt.equals(this.suspendedAt)) return this; + return new NodeVersion(hostname, zone, currentVersion, wantedVersion, suspendedAt); } @Override public String toString() { - return hostname + ": " + currentVersion + " -> " + wantedVersion + " [zone=" + zone + ", changedAt=" + changedAt + "]"; + return hostname + ": " + currentVersion + " -> " + wantedVersion + " [zone=" + zone + ", suspendedAt=" + suspendedAt.map(Instant::toString).orElse("") + "]"; } @Override @@ -87,12 +98,17 @@ public class NodeVersion { zone.equals(that.zone) && currentVersion.equals(that.currentVersion) && wantedVersion.equals(that.wantedVersion) && - changedAt.equals(that.changedAt); + suspendedAt.equals(that.suspendedAt); } @Override public int hashCode() { - return Objects.hash(hostname, zone, currentVersion, wantedVersion, changedAt); + return Objects.hash(hostname, zone, currentVersion, wantedVersion, suspendedAt); + } + + /** Returns whether this is changing (upgrading or downgrading) */ + private boolean changing() { + return !currentVersion.equals(wantedVersion); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java index 3ab96e03bcd..4ce0a35e96f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java @@ -7,6 +7,7 @@ import com.google.common.collect.ListMultimap; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -20,12 +21,10 @@ import java.util.function.Predicate; */ public class NodeVersions { - public static final NodeVersions EMPTY = new NodeVersions(ImmutableMap.of()); - private final ImmutableMap nodeVersions; - public NodeVersions(ImmutableMap nodeVersions) { - this.nodeVersions = Objects.requireNonNull(nodeVersions); + public NodeVersions(Map nodeVersions) { + this.nodeVersions = ImmutableMap.copyOf(Objects.requireNonNull(nodeVersions)); } public Map asMap() { @@ -48,7 +47,7 @@ public class NodeVersions { /** Returns a copy of this containing only node versions of given version */ public NodeVersions matching(Version version) { - return filter(nodeVersion -> nodeVersion.currentVersion().equals(version)); + return copyOf(nodeVersions.values(), nodeVersion -> nodeVersion.currentVersion().equals(version)); } /** Returns number of node versions in this */ @@ -56,31 +55,6 @@ public class NodeVersions { return nodeVersions.size(); } - /** Returns a copy of this containing only the given node versions */ - public NodeVersions with(List nodeVersions) { - var newNodeVersions = ImmutableMap.builder(); - for (var nodeVersion : nodeVersions) { - var existing = this.nodeVersions.get(nodeVersion.hostname()); - if (existing != null) { - newNodeVersions.put(nodeVersion.hostname(), existing.withCurrentVersion(nodeVersion.currentVersion(), - nodeVersion.changedAt()) - .withWantedVersion(nodeVersion.wantedVersion())); - } else { - newNodeVersions.put(nodeVersion.hostname(), nodeVersion); - } - } - return new NodeVersions(newNodeVersions.build()); - } - - private NodeVersions filter(Predicate predicate) { - var newNodeVersions = ImmutableMap.builder(); - for (var kv : nodeVersions.entrySet()) { - if (!predicate.test(kv.getValue())) continue; - newNodeVersions.put(kv.getKey(), kv.getValue()); - } - return new NodeVersions(newNodeVersions.build()); - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -94,4 +68,21 @@ public class NodeVersions { return Objects.hash(nodeVersions); } + public static NodeVersions copyOf(List nodeVersions) { + return copyOf(nodeVersions, (ignored) -> true); + } + + public static NodeVersions copyOf(Map nodeVersions) { + return new NodeVersions(nodeVersions); + } + + private static NodeVersions copyOf(Collection nodeVersions, Predicate predicate) { + var newNodeVersions = ImmutableMap.builder(); + for (var nodeVersion : nodeVersions) { + if (!predicate.test(nodeVersion)) continue; + newNodeVersions.put(nodeVersion.hostname(), nodeVersion); + } + return new NodeVersions(newNodeVersions.build()); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index 1773a9c122e..1dffd1383bd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -16,7 +16,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -59,9 +58,7 @@ public class OsVersionStatus { /** Compute the current OS versions in this system. This is expensive and should be called infrequently */ public static OsVersionStatus compute(Controller controller) { - var osVersionStatus = controller.osVersionStatus(); var osVersions = new HashMap>(); - var now = controller.clock().instant(); controller.osVersions().forEach(osVersion -> osVersions.put(osVersion, new ArrayList<>())); for (var application : SystemApplication.all()) { @@ -71,19 +68,16 @@ public class OsVersionStatus { .targetVersionsOf(zone.getId()) .osVersion(application.nodeType()) .orElse(Version.emptyVersion); - controller.serviceRegistry().configServer().nodeRepository() - .list(zone.getId(), application.id()).stream() - .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) - .map(node -> new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(), targetOsVersion, now)) - .forEach(nodeVersion -> { - var newNodeVersion = osVersionStatus.of(nodeVersion.hostname()) - .map(nv -> nv.withCurrentVersion(nodeVersion.currentVersion(), now) - .withWantedVersion(nodeVersion.wantedVersion())) - .orElse(nodeVersion); - var version = new OsVersion(newNodeVersion.currentVersion(), zone.getCloudName()); - osVersions.putIfAbsent(version, new ArrayList<>()); - osVersions.get(version).add(newNodeVersion); - }); + + for (var node : controller.serviceRegistry().configServer().nodeRepository().list(zone.getId(), application.id())) { + if (!OsUpgrader.eligibleForUpgrade(node, application)) continue; + var suspendedAt = node.suspendedSince(); + var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(), + targetOsVersion, suspendedAt); + var osVersion = new OsVersion(nodeVersion.currentVersion(), zone.getCloudName()); + osVersions.putIfAbsent(osVersion, new ArrayList<>()); + osVersions.get(osVersion).add(nodeVersion); + } } } @@ -98,15 +92,6 @@ public class OsVersionStatus { return new OsVersionStatus(newOsVersions.build()); } - /** Returns version of node identified by given host name */ - private Optional of(HostName hostname) { - return versions.values().stream() - .map(nodeVersions -> nodeVersions.asMap().get(hostname)) - .map(Optional::ofNullable) - .flatMap(Optional::stream) - .findFirst(); - } - private static List zonesToUpgrade(Controller controller) { return controller.zoneRegistry().osUpgradePolicies().stream() .flatMap(upgradePolicy -> upgradePolicy.asList().stream()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 0868d7ca695..7618e8210e3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -8,23 +8,15 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; -import com.yahoo.vespa.hosted.controller.deployment.JobStatus; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.maintenance.SystemUpgrader; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -149,10 +141,7 @@ public class VersionStatus { } private static NodeVersions findSystemApplicationVersions(Controller controller) { - var nodeVersions = controller.versionStatus().systemVersion() - .map(VespaVersion::nodeVersions) - .orElse(NodeVersions.EMPTY); - var newNodeVersions = new ArrayList(); + var nodeVersions = new LinkedHashMap(); for (var zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) { for (var application : SystemApplication.all()) { var nodes = controller.serviceRegistry().configServer().nodeRepository() @@ -165,15 +154,16 @@ public class VersionStatus { log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone.getId() + " has not converged"); } - var now = controller.clock().instant(); for (var node : nodes) { // Only use current node version if config has converged - Version version = configConverged ? node.currentVersion() : controller.systemVersion(); - newNodeVersions.add(new NodeVersion(node.hostname(), zone.getId(), version, node.wantedVersion(), now)); + var version = configConverged ? node.currentVersion() : controller.systemVersion(); + var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), version, node.wantedVersion(), + node.suspendedSince()); + nodeVersions.put(nodeVersion.hostname(), nodeVersion); } } } - return nodeVersions.with(newNodeVersions); + return NodeVersions.copyOf(nodeVersions); } private static ListMultimap findControllerVersions(Controller controller) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 775eb2a4d75..189bd5156e3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -223,7 +223,7 @@ public final class ControllerTester { if (!application.hasApplicationPackage()) { configServer().nodeRepository().upgrade(zone.getId(), application.nodeType(), version); } - configServer().setVersion(application.id(), zone.getId(), version); + configServer().setVersion(version, application.id(), zone.getId()); configServer().convergeServices(application.id(), zone.getId()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index d86d2960c74..07c643070a0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -201,9 +201,9 @@ public class InternalStepRunnerTest { // Node is down too long in system test, and no nodes go down in staging. tester.runner().run(); - tester.configServer().setVersion(app.testerId().id(), JobType.systemTest.zone(system()), tester.controller().systemVersion()); + tester.configServer().setVersion(tester.controller().systemVersion(), app.testerId().id(), JobType.systemTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); - tester.configServer().setVersion(app.testerId().id(), JobType.stagingTest.zone(system()), tester.controller().systemVersion()); + tester.configServer().setVersion(tester.controller().systemVersion(), app.testerId().id(), JobType.stagingTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); @@ -377,7 +377,7 @@ public class InternalStepRunnerTest { assertEquals(applicationPackage.hash(), tester.configServer().application(app.instanceId(), zone).get().applicationPackage().hash()); assertEquals(otherPackage.hash(), tester.configServer().application(app.instanceId(), JobType.perfUsEast3.zone(system())).get().applicationPackage().hash()); - tester.configServer().setVersion(app.instanceId(), zone, version); + tester.configServer().setVersion(version, app.instanceId(), zone); tester.runner().run(); assertEquals(1, tester.jobs().active().size()); assertEquals(version, tester.instance(app.instanceId()).deployments().get(zone).version()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index d5a71d5adca..0fb2451ccba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -178,28 +178,27 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } /** Set version for an application in a given zone */ - public void setVersion(ApplicationId application, ZoneId zone, Version version) { - setVersion(application, zone, version, -1, false); + public void setVersion(Version version, ApplicationId application, ZoneId zone) { + setVersion(zone, nodeRepository.list(zone, application), version, false); } /** Set version for nodeCount number of nodes in application in a given zone */ - public void setVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount) { - setVersion(application, zone, version, nodeCount, false); + public void setVersion(Version version, List nodes, ZoneId zone) { + setVersion(zone, nodes, version, false); } /** Set OS version for an application in a given zone */ - public void setOsVersion(ApplicationId application, ZoneId zone, Version version) { - setOsVersion(application, zone, version, -1); + public void setOsVersion(Version version, ApplicationId application, ZoneId zone) { + setVersion(zone, nodeRepository.list(zone, application), version, true); } /** Set OS version for an application in a given zone */ - public void setOsVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount) { - setVersion(application, zone, version, nodeCount, true); + public void setOsVersion(Version version, List nodes, ZoneId zone) { + setVersion(zone, nodes, version, true); } - private void setVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount, boolean osVersion) { - int n = 0; - for (Node node : nodeRepository().list(zone, application)) { + private void setVersion(ZoneId zone, List nodes, Version version, boolean osVersion) { + for (var node : nodes) { Node newNode; if (osVersion) { newNode = new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build(); @@ -207,7 +206,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer newNode = new Node.Builder(node).currentVersion(version).wantedVersion(version).build(); } nodeRepository().putNodes(zone, newNode); - if (++n == nodeCount) break; } } @@ -397,8 +395,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public void deactivate(DeploymentId deployment) throws NotFoundException { ApplicationId applicationId = deployment.applicationId(); - nodeRepository().removeByHostname(deployment.zoneId(), - nodeRepository().list(deployment.zoneId(), applicationId)); + nodeRepository().removeNodes(deployment.zoneId(), + nodeRepository().list(deployment.zoneId(), applicationId)); if ( ! applications.containsKey(deployment)) throw new NotFoundException("No application with id " + applicationId + " exists, cannot deactivate"); applications.remove(deployment); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index c00705149e9..e2482fc9174 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; @@ -22,12 +23,15 @@ import org.junit.Test; import java.time.Duration; import java.util.Comparator; import java.util.List; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author mortent @@ -226,11 +230,11 @@ public class MetricsReporterTest { public void nodes_failing_system_upgrade() { var tester = new ControllerTester(); var reporter = createReporter(tester.controller()); - var zone1 = ZoneApiMock.fromId("prod.eu-west-1"); - tester.zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(zone1)); + var zone = ZoneId.from("prod.eu-west-1"); + tester.zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(ZoneApiMock.from(zone))); var systemUpgrader = new SystemUpgrader(tester.controller(), Duration.ofDays(1), new JobControl(tester.curator())); - tester.configServer().bootstrap(List.of(zone1.getId()), SystemApplication.configServer); + tester.configServer().bootstrap(List.of(zone), SystemApplication.configServer); // System on initial version var version0 = Version.fromString("7.0"); @@ -247,25 +251,37 @@ public class MetricsReporterTest { // 30 minutes pass and nothing happens tester.clock().advance(Duration.ofMinutes(30)); - tester.computeVersionStatus(); - reporter.maintain(); + runAll(tester::computeVersionStatus, reporter); assertEquals(0, getNodesFailingUpgrade()); // 1/3 nodes upgrade within timeout - assertEquals("Wanted version is raised for all nodes", version, - tester.configServer().nodeRepository().list(zone1.getId(), SystemApplication.configServer.id()).stream() - .map(Node::wantedVersion).min(Comparator.naturalOrder()).get()); - tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version, 1); - tester.clock().advance(Duration.ofMinutes(60).plus(Duration.ofSeconds(1))); - tester.computeVersionStatus(); - reporter.maintain(); + var hosts = tester.configServer().nodeRepository().list(zone, SystemApplication.configServer.id()); + assertEquals("Wanted version is raised for all nodes", version, hosts.stream() + .map(Node::wantedVersion) + .min(Comparator.naturalOrder()) + .get()); + suspend(hosts, zone, tester); + var firstHost = hosts.get(0); + upgradeTo(version, List.of(firstHost), zone, tester); + + // 2/3 spend their budget and are reported as failures + tester.clock().advance(Duration.ofHours(1)); + runAll(tester::computeVersionStatus, reporter); assertEquals(2, getNodesFailingUpgrade()); - - // 3/3 nodes upgrade - tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version); - tester.computeVersionStatus(); - reporter.maintain(); + assertEquals("Change duration is reset for completed upgrade", Duration.ZERO, + getChangeDuration(MetricsReporter.PLATFORM_CHANGE_DURATION, firstHost.hostname())); + for (var host : hosts.subList(1, hosts.size())) { + assertEquals(Duration.ofHours(1), getChangeDuration(MetricsReporter.PLATFORM_CHANGE_DURATION, + host.hostname())); + } + + // Remaining nodes eventually upgrade + upgradeTo(version, hosts.subList(1, 3), zone, tester); + runAll(tester::computeVersionStatus, reporter); assertEquals(0, getNodesFailingUpgrade()); + for (var host : hosts) { + assertEquals(Duration.ZERO, getChangeDuration(MetricsReporter.PLATFORM_CHANGE_DURATION, host.hostname())); + } assertEquals(version, tester.controller().systemVersion()); } } @@ -274,64 +290,131 @@ public class MetricsReporterTest { public void nodes_failing_os_upgrade() { var tester = new ControllerTester(); var reporter = createReporter(tester.controller()); - var zone = ZoneApiMock.fromId("prod.eu-west-1"); + var zone = ZoneId.from("prod.eu-west-1"); var cloud = CloudName.defaultName(); - tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(zone)); + tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(ZoneApiMock.from(zone))); var osUpgrader = new OsUpgrader(tester.controller(), Duration.ofDays(1), new JobControl(tester.curator()), CloudName.defaultName()); var statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator())); - tester.configServer().bootstrap(List.of(zone.getId()), SystemApplication.configServerHost, SystemApplication.tenantHost); + tester.configServer().bootstrap(List.of(zone), SystemApplication.configServerHost, SystemApplication.tenantHost); // All nodes upgrade to initial OS version var version0 = Version.fromString("8.0"); tester.controller().upgradeOsIn(cloud, version0, false); osUpgrader.maintain(); - tester.configServer().setOsVersion(SystemApplication.tenantHost.id(), zone.getId(), version0); - tester.configServer().setOsVersion(SystemApplication.configServerHost.id(), zone.getId(), version0); - statusUpdater.maintain(); - reporter.maintain(); + tester.configServer().setOsVersion(version0, SystemApplication.tenantHost.id(), zone); + tester.configServer().setOsVersion(version0, SystemApplication.configServerHost.id(), zone); + runAll(statusUpdater, reporter); assertEquals(0, getNodesFailingOsUpgrade()); for (var version : List.of(Version.fromString("8.1"), Version.fromString("8.2"))) { // System starts upgrading to next OS version tester.controller().upgradeOsIn(cloud, version, false); - osUpgrader.maintain(); - statusUpdater.maintain(); - reporter.maintain(); + runAll(osUpgrader, statusUpdater, reporter); assertEquals(0, getNodesFailingOsUpgrade()); - // 30 minutes pass and nothing happens - tester.clock().advance(Duration.ofMinutes(30)); - statusUpdater.maintain(); - reporter.maintain(); + // Over 30 minutes pass and nothing happens + tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + runAll(statusUpdater, reporter); assertEquals(0, getNodesFailingOsUpgrade()); - // 2/6 nodes upgrade within timeout + // Nodes are told to upgrade, but do not suspend yet assertEquals("Wanted OS version is raised for all nodes", version, - tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.tenantHost.id()).stream() + tester.configServer().nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream() .map(Node::wantedOsVersion).min(Comparator.naturalOrder()).get()); - tester.configServer().setOsVersion(SystemApplication.tenantHost.id(), zone.getId(), version, 2); - tester.clock().advance(Duration.ofMinutes(30 * 3 /* time allowance * node count */).plus(Duration.ofSeconds(1))); - statusUpdater.maintain(); - reporter.maintain(); - assertEquals(4, getNodesFailingOsUpgrade()); + assertTrue("No nodes are suspended", tester.controller().serviceRegistry().configServer() + .nodeRepository().list(zone).stream() + .noneMatch(node -> node.serviceState() == Node.ServiceState.allowedDown)); - // 5/6 nodes upgrade - tester.configServer().setOsVersion(SystemApplication.tenantHost.id(), zone.getId(), version); - tester.configServer().setOsVersion(SystemApplication.configServerHost.id(), zone.getId(), version, 2); - statusUpdater.maintain(); - reporter.maintain(); + // Another 30 minutes pass + tester.clock().advance(Duration.ofMinutes(30)); + runAll(statusUpdater, reporter); + assertEquals(0, getNodesFailingOsUpgrade()); + + // 3/6 hosts suspend + var hosts = tester.configServer().nodeRepository().list(zone); + var suspendedHosts = hosts.subList(0, 3); + suspend(suspendedHosts, zone, tester); + runAll(statusUpdater, reporter); + assertEquals(0, getNodesFailingOsUpgrade()); + + // Two hosts upgrade after 20 minutes + var hostsUpgraded = suspendedHosts.subList(0, 2); + tester.clock().advance(Duration.ofMinutes(20)); + runAll(statusUpdater, reporter); + for (var host : suspendedHosts) { + assertEquals(Duration.ofMinutes(20), getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, host.hostname())); + } + upgradeOsTo(version, hostsUpgraded, zone, tester); + runAll(statusUpdater, reporter); + assertEquals(0, getNodesFailingOsUpgrade()); + for (var host : hostsUpgraded) { + assertEquals(Duration.ZERO, getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, host.hostname())); + } + + // One host consumes budget without upgrading + var brokenHost = suspendedHosts.get(2); + tester.clock().advance(Duration.ofMinutes(15)); + runAll(statusUpdater, reporter); assertEquals(1, getNodesFailingOsUpgrade()); + assertEquals(Duration.ofMinutes(35), getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, + brokenHost.hostname())); - // Final node upgrades - tester.configServer().setOsVersion(SystemApplication.configServerHost.id(), zone.getId(), version); - statusUpdater.maintain(); - reporter.maintain(); + // Host eventually upgrades and is no longer reported + upgradeOsTo(version, List.of(brokenHost), zone, tester); + runAll(statusUpdater, reporter); + assertEquals(0, getNodesFailingOsUpgrade()); + assertEquals(Duration.ZERO, getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, brokenHost.hostname())); + + // Remaining hosts suspend and upgrade successfully + var remainingHosts = hosts.subList(3, hosts.size()); + suspend(remainingHosts, zone, tester); + upgradeOsTo(version, remainingHosts, zone, tester); + runAll(statusUpdater, reporter); assertEquals(0, getNodesFailingOsUpgrade()); + for (var host : hosts) { + assertEquals(Duration.ZERO, getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, host.hostname())); + } } } + private void runAll(Runnable... runnables) { + for (var r : runnables) r.run(); + } + + private void upgradeTo(Version version, List nodes, ZoneId zone, ControllerTester tester) { + tester.configServer().setVersion(version, nodes, zone); + resume(nodes, zone, tester); + } + + private void upgradeOsTo(Version version, List nodes, ZoneId zone, ControllerTester tester) { + tester.configServer().setOsVersion(version, nodes, zone); + resume(nodes, zone, tester); + } + + private void resume(List nodes, ZoneId zone, ControllerTester tester) { + updateNodes(nodes, (builder) -> builder.serviceState(Node.ServiceState.expectedUp).suspendedSince(null), + zone, tester); + } + + private void suspend(List nodes, ZoneId zone, ControllerTester tester) { + var now = tester.clock().instant(); + updateNodes(nodes, (builder) -> builder.serviceState(Node.ServiceState.allowedDown).suspendedSince(now), + zone, tester); + } + + private void updateNodes(List nodes, UnaryOperator builderOps, ZoneId zone, + ControllerTester tester) { + var currentNodes = tester.configServer().nodeRepository().list(zone, nodes.stream() + .map(Node::hostname) + .collect(Collectors.toList())); + var updatedNodes = currentNodes.stream() + .map(node -> builderOps.apply(new Node.Builder(node)).build()) + .collect(Collectors.toList()); + tester.configServer().nodeRepository().putNodes(zone, updatedNodes); + } + private Duration getAverageDeploymentDuration(ApplicationId id) { return Duration.ofSeconds(getMetric(MetricsReporter.DEPLOYMENT_AVERAGE_DURATION, id).longValue()); } @@ -345,7 +428,13 @@ public class MetricsReporterTest { } private int getNodesFailingUpgrade() { - return metrics.getMetric(MetricsReporter.NODES_FAILING_SYSTEM_UPGRADE).intValue(); + return metrics.getMetric(MetricsReporter.NODES_FAILING_PLATFORM_UPGRADE).intValue(); + } + + private Duration getChangeDuration(String metric, HostName hostname) { + return metrics.getMetric((dimensions) -> hostname.value().equals(dimensions.get("host")), metric) + .map(n -> Duration.ofSeconds(n.longValue())) + .orElseThrow(() -> new IllegalArgumentException("Expected to find metric for " + hostname)); } private int getNodesFailingOsUpgrade() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java index e585547e921..efdf412ab77 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java @@ -14,6 +14,7 @@ import org.junit.Test; import java.time.Instant; import java.util.List; +import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -23,18 +24,18 @@ import static org.junit.Assert.assertEquals; public class OsVersionStatusSerializerTest { @Test - public void test_serialization() { + public void serialization() { Version version1 = Version.fromString("7.1"); Version version2 = Version.fromString("7.2"); var versions = ImmutableMap.builder(); - versions.put(new OsVersion(version1, CloudName.defaultName()), NodeVersions.EMPTY.with(List.of( - new NodeVersion(HostName.from("node1"), ZoneId.from("prod", "us-west"), version1, version2, Instant.ofEpochMilli(1)), - new NodeVersion(HostName.from("node2"), ZoneId.from("prod", "us-east"), version1, version2, Instant.ofEpochMilli(2)) + versions.put(new OsVersion(version1, CloudName.defaultName()), NodeVersions.copyOf(List.of( + new NodeVersion(HostName.from("node1"), ZoneId.from("prod", "us-west"), version1, version2, Optional.of(Instant.ofEpochMilli(11))), + new NodeVersion(HostName.from("node2"), ZoneId.from("prod", "us-east"), version1, version2, Optional.of(Instant.ofEpochMilli(22))) ))); - versions.put(new OsVersion(version2, CloudName.defaultName()), NodeVersions.EMPTY.with(List.of( - new NodeVersion(HostName.from("node3"), ZoneId.from("prod", "us-west"), version2, version2, Instant.ofEpochMilli(3)), - new NodeVersion(HostName.from("node4"), ZoneId.from("prod", "us-east"), version2, version2, Instant.ofEpochMilli(4)) + versions.put(new OsVersion(version2, CloudName.defaultName()), NodeVersions.copyOf(List.of( + new NodeVersion(HostName.from("node3"), ZoneId.from("prod", "us-west"), version2, version2, Optional.of(Instant.ofEpochMilli(33))), + new NodeVersion(HostName.from("node4"), ZoneId.from("prod", "us-east"), version2, version2, Optional.of(Instant.ofEpochMilli(44))) ))); OsVersionStatusSerializer serializer = new OsVersionStatusSerializer(new OsVersionSerializer(), new NodeVersionSerializer()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index 0c1f94d90cf..41d59239321 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; @@ -14,8 +12,8 @@ import org.junit.Test; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.Optional; import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; @@ -31,10 +29,10 @@ public class VersionStatusSerializerTest { Version version = Version.fromString("5.0"); vespaVersions.add(new VespaVersion(version, "dead", Instant.now(), false, false, true, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"), - Instant.ofEpochMilli(123), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); vespaVersions.add(new VespaVersion(version, "cafe", Instant.now(), true, true, false, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"), - Instant.ofEpochMilli(456), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); VersionStatus status = new VersionStatus(vespaVersions); VersionStatusSerializer serializer = new VersionStatusSerializer(new NodeVersionSerializer()); VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status)); @@ -55,12 +53,12 @@ public class VersionStatusSerializerTest { } - private static NodeVersions nodeVersions(Version version, Version wantedVersion, Instant changedAt, String... hostnames) { + private static NodeVersions nodeVersions(Version version, Version wantedVersion, String... hostnames) { var nodeVersions = new ArrayList(); for (var hostname : hostnames) { - nodeVersions.add(new NodeVersion(HostName.from(hostname), ZoneId.from("prod", "us-north-1"), version, wantedVersion, changedAt)); + nodeVersions.add(new NodeVersion(HostName.from(hostname), ZoneId.from("prod", "us-north-1"), version, wantedVersion, Optional.empty())); } - return NodeVersions.EMPTY.with(nodeVersions); + return NodeVersions.copyOf(nodeVersions); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index c8036676fa9..32a77137182 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -19,9 +19,9 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; import java.io.File; -import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * @author bratseth @@ -87,8 +87,8 @@ public class DeploymentApiTest extends ControllerContainerTest { version.isControllerVersion(), version.isSystemVersion(), version.isReleased(), - NodeVersions.EMPTY.with(List.of(new NodeVersion(HostName.from("config1.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Instant.EPOCH), - new NodeVersion(HostName.from("config2.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Instant.EPOCH))), + NodeVersions.copyOf(List.of(new NodeVersion(HostName.from("config1.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Optional.empty()), + new NodeVersion(HostName.from("config2.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Optional.empty()))), version.confidence() ); } -- cgit v1.2.3 From 000603b162bf8c28073fde5d56d6a3ee46e48574 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Wed, 22 Apr 2020 14:33:57 +0200 Subject: Increase streaming search timeout trace sampling probabilities --- .../com/yahoo/vespa/streamingvisitors/TracingOptions.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java index 3c96b00d628..b9e19d6a1b6 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java @@ -15,12 +15,12 @@ import java.util.concurrent.TimeUnit; * Encapsulates all trace-related components and options used by the streaming search Searcher. * * Provides a DEFAULT static instance which has the following characteristics: - * - Approximately 1 query every 2 seconds is traced + * - Approximately 1 query every second is traced * - Trace level is set to 7 for traced queries - * - Only emits traces for queries that have timed out and where the elapsed time is at least 5x + * - Only emits traces for queries that have timed out and where the elapsed time is at least 2x * of the timeout specified in the query itself * - Emits traces to the Vespa log - * - Only 1 trace every 10 seconds may be emitted to the log + * - Only 2 traces every 10 seconds may be emitted to the log */ public class TracingOptions { @@ -50,11 +50,11 @@ public class TracingOptions { public static final TracingOptions DEFAULT; public static final int DEFAULT_TRACE_LEVEL_OVERRIDE = 7; // TODO determine appropriate trace level // Traces won't be exported unless the query has timed out with a duration that is > timeout * multiplier - public static final double TRACE_TIMEOUT_MULTIPLIER_THRESHOLD = 5.0; + public static final double TRACE_TIMEOUT_MULTIPLIER_THRESHOLD = 2.0; static { - SamplingStrategy queryTraceSampler = ProbabilisticSampleRate.withSystemDefaults(0.5); - SamplingStrategy logExportSampler = MaxSamplesPerPeriod.withSteadyClock(TimeUnit.SECONDS.toNanos(10), 1); + SamplingStrategy queryTraceSampler = ProbabilisticSampleRate.withSystemDefaults(1); + SamplingStrategy logExportSampler = MaxSamplesPerPeriod.withSteadyClock(TimeUnit.SECONDS.toNanos(10), 2); TraceExporter traceExporter = new SamplingTraceExporter(new LoggingTraceExporter(), logExportSampler); DEFAULT = new TracingOptions(queryTraceSampler, traceExporter, System::nanoTime, DEFAULT_TRACE_LEVEL_OVERRIDE, TRACE_TIMEOUT_MULTIPLIER_THRESHOLD); -- cgit v1.2.3 From 2afc5ebb9c65cb1aaf0f8c338dffa81f1f74a3ec Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 22 Apr 2020 14:37:50 +0200 Subject: Revert "Reduce timeout for nodesDown and noNodeDown in cd" --- .../yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 1f2d503326a..c49c56c4e82 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -941,8 +941,8 @@ public class InternalStepRunner implements StepRunner { Duration endpoint() { return Duration.ofMinutes(15); } Duration endpointCertificate() { return Duration.ofMinutes(20); } Duration tester() { return Duration.ofMinutes(30); } - Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 20 : 60); } - Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 20 : 120); } + Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 60); } + Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 120); } Duration testerCertificate() { return Duration.ofMinutes(300); } } -- cgit v1.2.3 From 6bad70cf6e5a91d895c6f0da3496902c4f93e341 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 22 Apr 2020 14:53:35 +0200 Subject: Extend semantic lifetime of view resolver variables in searchcore query test. --- searchcore/src/tests/proton/matching/query_test.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 96e1eb3cc22..24e1e886351 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -546,8 +546,9 @@ void Test::requireThatNearIteratorsCanBeBuilt() { builder.addStringTerm(string_term, field, 1, Weight(2)); builder.addStringTerm(prefix_term, field, 1, Weight(2)); Node::UP node = builder.build(); - ResolveViewVisitor resolver(ViewResolver(), plain_index_env); - node->accept(resolver); + ViewResolver resolver; + ResolveViewVisitor visitor(resolver, plain_index_env); + node->accept(visitor); ASSERT_TRUE(node.get()); FakeSearchContext context(8); @@ -569,8 +570,9 @@ void Test::requireThatONearIteratorsCanBeBuilt() { builder.addStringTerm(string_term, field, 1, Weight(2)); builder.addStringTerm(prefix_term, field, 1, Weight(2)); Node::UP node = builder.build(); - ResolveViewVisitor resolver(ViewResolver(), plain_index_env); - node->accept(resolver); + ViewResolver resolver; + ResolveViewVisitor visitor(resolver, plain_index_env); + node->accept(visitor); ASSERT_TRUE(node.get()); FakeSearchContext context(8); @@ -593,8 +595,9 @@ void Test::requireThatPhraseIteratorsCanBeBuilt() { builder.addStringTerm(prefix_term, field, 1, Weight(2)); builder.addStringTerm(suffix_term, field, 1, Weight(2)); Node::UP node = builder.build(); - ResolveViewVisitor resolver(ViewResolver(), plain_index_env); - node->accept(resolver); + ViewResolver resolver; + ResolveViewVisitor visitor(resolver, plain_index_env); + node->accept(visitor); ASSERT_TRUE(node.get()); FakeSearchContext context(9); -- cgit v1.2.3 From d2090d1f4b4a6c56fb528d819af7ee5b680f3ef7 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 27 Sep 2019 14:47:45 +0200 Subject: Create DiskCleanup --- .../maintenance/disk/CoredumpCleanupRule.java | 106 +++++++++++++++ .../node/admin/maintenance/disk/DiskCleanup.java | 67 ++++++++++ .../admin/maintenance/disk/DiskCleanupRule.java | 31 +++++ .../admin/maintenance/disk/LinearCleanupRule.java | 48 +++++++ .../node/admin/maintenance/disk/package-info.java | 5 + .../node/admin/task/util/file/FileFinder.java | 2 +- .../maintenance/disk/CoredumpCleanupRuleTest.java | 103 +++++++++++++++ .../admin/maintenance/disk/DiskCleanupTest.java | 142 +++++++++++++++++++++ .../maintenance/disk/LinearCleanupRuleTest.java | 55 ++++++++ 9 files changed, 558 insertions(+), 1 deletion(-) create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java new file mode 100644 index 00000000000..2ad887514e5 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java @@ -0,0 +1,106 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; + +import java.nio.file.Path; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.temporal.ChronoField; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameStartsWith; + +/** + * @author freva + */ +public class CoredumpCleanupRule { + + private static final Comparator CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR = Comparator + .comparing((FileAttributes fa) -> !fa.filename().contains("vespa-")) + .thenComparing(FileAttributes::lastModifiedTime); + + public static DiskCleanupRule forContainer(Path containerCrashPath) { + return new ContainerCoredumpCleanupRule(containerCrashPath); + } + + public static DiskCleanupRule forHost(Path processedCoredumpsPath) { + return new HostCoredumpCleanupRule(processedCoredumpsPath); + } + + /** Assigns MEDIUM priority to the oldest, unprocessed coredump and HIGHEST for the remaining */ + private static class ContainerCoredumpCleanupRule implements DiskCleanupRule { + private final Path containerCrashPath; + + private ContainerCoredumpCleanupRule(Path containerCrashPath) { + this.containerCrashPath = containerCrashPath; + } + + @Override + public Collection prioritize() { + List fileAttributes = FileFinder.files(containerCrashPath) + .maxDepth(1).stream() + .sorted(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR) + .collect(Collectors.toList()); + + return mapFirstAndRemaining(fileAttributes, Priority.MEDIUM, Priority.HIGHEST).collect(Collectors.toList()); + } + } + + /** Assigns MEDIUM priority to the first coredump of the day for each container, HIGH for the remaining */ + private static class HostCoredumpCleanupRule implements DiskCleanupRule { + private final Path processedCoredumpsPath; + + private HostCoredumpCleanupRule(Path processedCoredumpsPath) { + this.processedCoredumpsPath = processedCoredumpsPath; + } + + @Override + public Collection prioritize() { + Map> fileAttributesByContainerDay = FileFinder.files(processedCoredumpsPath) + .match(nameStartsWith(CoredumpHandler.COREDUMP_FILENAME_PREFIX)) + .stream() + .sorted(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR) + .collect(Collectors.groupingBy( + // Group FileAttributes by string [container-name]_[day of year], e.g. zt00534-v6-2_234 + fa -> containerNameFromProcessedCoredumpPath(fa.path()) + "_" + dayOfYear(fa.lastModifiedTime()), + Collectors.collectingAndThen( + Collectors.toCollection(ArrayList::new), + l -> { l.sort(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR); return l; } ))); + + return fileAttributesByContainerDay.values().stream() + .flatMap(fa -> mapFirstAndRemaining(fa, Priority.MEDIUM, Priority.HIGH)) + .collect(Collectors.toList()); + } + } + + /** + * Maps list of FileAttributes into list of PrioritizedFileAttributes where the first FileAttribute is given + * {@code first} priority, while the remaining FileAttributes are given {@code remaining} priority */ + private static Stream mapFirstAndRemaining(List fileAttributes, Priority first, Priority remaining) { + return IntStream.range(0, fileAttributes.size()) + .mapToObj(i -> new PrioritizedFileAttributes(fileAttributes.get(i), i == 0 ? first : remaining)); + } + + /** Extracts container-name from path under processed-coredumps or empty string */ + private static String containerNameFromProcessedCoredumpPath(Path path) { + if (path.getNameCount() < 3) return ""; // Path is too short + return path.getName(path.getNameCount() - 3).toString(); + } + + /** Returns day number of the year (1-365 (or 366 for leap years)) */ + private static int dayOfYear(Instant instant) { + return instant.atOffset(ZoneOffset.UTC).get(ChronoField.DAY_OF_YEAR); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java new file mode 100644 index 00000000000..a9c1f8a9600 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java @@ -0,0 +1,67 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.logging.Logger; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author freva + */ +public class DiskCleanup { + + private static final Logger logger = Logger.getLogger(DiskCleanup.class.getName()); + private static final char[] UNITS = "kMGTPE".toCharArray(); + private static final Comparator PRIORITIZED_FILE_ATTRIBUTES_COMPARATOR = Comparator + .comparing(PrioritizedFileAttributes::priority) + .thenComparingLong(f -> f.fileAttributes().size()) + .reversed(); + + public boolean cleanup(TaskContext context, List rules, long bytesToRemove) { + if (bytesToRemove <= 0) return false; + + long[] btr = new long[] { bytesToRemove }; + List deletedPaths = new ArrayList<>(); + try { + rules.stream() + .flatMap(rule -> rule.prioritize().stream()) + .sorted(PRIORITIZED_FILE_ATTRIBUTES_COMPARATOR) + .takeWhile(fa -> btr[0] > 0) + .forEach(pfa -> { + if (uncheck(() -> Files.deleteIfExists(pfa.fileAttributes().path()))) { + btr[0] -= pfa.fileAttributes().size(); + deletedPaths.add(pfa.fileAttributes().path()); + } + }); + + } finally { + String wantedDeleteSize = bytesToDisplayCount(bytesToRemove); + String deletedSize = bytesToDisplayCount(bytesToRemove - btr[0]); + if (deletedPaths.size() > 20) { + context.log(logger, "Deleted %d files (%s) because disk was getting full", deletedPaths.size(), deletedSize); + } else if (deletedPaths.size() > 0) { + context.log(logger, "Deleted %s because disk was getting full from: %s", deletedSize, deletedPaths); + } else { + context.log(logger, "Wanted to delete %s, but failed to find any files to delete", wantedDeleteSize); + } + } + + return !deletedPaths.isEmpty(); + } + + static String bytesToDisplayCount(long bytes) { + if (bytes < 1000) return bytes + " bytes"; + + int unit = -1; + for (; bytes >= 1000; unit++) bytes /= 1000; + return bytes + " " + UNITS[unit] + "B"; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java new file mode 100644 index 00000000000..a4c8229d0d7 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; + +import java.util.Collection; + +/** + * @author freva + */ +public interface DiskCleanupRule { + + Collection prioritize(); + + enum Priority { + LOWEST, LOW, MEDIUM, HIGH, HIGHEST + } + + class PrioritizedFileAttributes { + private final FileFinder.FileAttributes fileAttributes; + private final Priority priority; + + public PrioritizedFileAttributes(FileFinder.FileAttributes fileAttributes, Priority priority) { + this.fileAttributes = fileAttributes; + this.priority = priority; + } + + public Priority priority() { return priority; } + public FileFinder.FileAttributes fileAttributes() { return fileAttributes; } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java new file mode 100644 index 00000000000..3ef4e2e4f6a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java @@ -0,0 +1,48 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; + +/** + * Prioritizes files by first scoring them with the given scoring function and then mapping the scores to a + * priority within the given range. + * The priority room is evenly split between given lowest and highest priority for range [0, 1.0). Scores below 0 + * are assigned lowest, while scores at or higher than 1 are assigned highest priority. + * + * Typical use-case is for log files. The scoring function calculates the file age and normalizes it by dividing it + * by expected max age of log files. The oldest logs will then by prioritized by highest given priority. + * + * @author freva + */ +public class LinearCleanupRule implements DiskCleanupRule { + private final Supplier> lister; + private final Function prioritizer; + + public LinearCleanupRule(Supplier> lister, + Function scorer, Priority lowest, Priority highest) { + if (lowest.ordinal() > highest.ordinal()) + throw new IllegalArgumentException("Lowest priority: " + lowest + " is higher than highest priority: " + highest); + + this.lister = lister; + + Priority[] values = Priority.values(); + int range = highest.ordinal() - lowest.ordinal() + 1; + this.prioritizer = fa -> { + int ordinal = (int) (lowest.ordinal() + scorer.apply(fa) * range); + return values[Math.max(lowest.ordinal(), Math.min(highest.ordinal(), ordinal))]; + }; + } + + @Override + public Collection prioritize() { + return lister.get().stream() + .map(fa -> new PrioritizedFileAttributes(fa, prioritizer.apply(fa))) + .collect(Collectors.toList()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java new file mode 100644 index 00000000000..dff40015522 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java @@ -0,0 +1,5 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java index f0b064373d2..a41281d0479 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java @@ -193,7 +193,7 @@ public class FileFinder { private final Path path; private final BasicFileAttributes attributes; - FileAttributes(Path path, BasicFileAttributes attributes) { + public FileAttributes(Path path, BasicFileAttributes attributes) { this.path = path; this.attributes = attributes; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java new file mode 100644 index 00000000000..e8ca2ff8491 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java @@ -0,0 +1,103 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class CoredumpCleanupRuleTest { + + private final FileSystem fileSystem = TestFileSystem.create(); + + @Test + public void for_container_test() throws IOException { + Path path = fileSystem.getPath("/test/path"); + DiskCleanupRule rule = CoredumpCleanupRule.forContainer(path); + + assertPriorities(rule, Map.of()); + + createFile(path.resolve("core1"), Instant.ofEpochSecond(232)); + assertPriorities(rule, Map.of("/test/path/core1", Priority.MEDIUM)); + + createFile(path.resolve("core2"), Instant.ofEpochSecond(123)); + assertPriorities(rule, Map.of( + "/test/path/core2", Priority.MEDIUM, + "/test/path/core1", Priority.HIGHEST)); + + createFile(path.resolve("vespa-proton-bin.core.325"), Instant.ofEpochSecond(456)); + createFile(path.resolve("vespa-distributor.core.764"), Instant.ofEpochSecond(256)); + var expected = Map.of( + "/test/path/core2", Priority.HIGHEST, + "/test/path/core1", Priority.HIGHEST, + "/test/path/vespa-proton-bin.core.325", Priority.HIGHEST, + "/test/path/vespa-distributor.core.764", Priority.MEDIUM); + assertPriorities(rule, expected); + + // processing core has no effect on this + Files.createDirectories(path.resolve("processing/abcd-1234")); + createFile(path.resolve("processing/abcd-1234/core5"), Instant.ofEpochSecond(67)); + assertPriorities(rule, expected); + } + + @Test + public void for_host_test() throws IOException { + Path path = fileSystem.getPath("/test/path"); + DiskCleanupRule rule = CoredumpCleanupRule.forHost(path); + + assertPriorities(rule, Map.of()); + + createFile(path.resolve("h123a/abcd-1234/dump_core1"), Instant.parse("2020-04-21T19:21:00Z")); + createFile(path.resolve("h123a/abcd-1234/metadata.json"), Instant.parse("2020-04-21T19:26:00Z")); + assertPriorities(rule, Map.of("/test/path/h123a/abcd-1234/dump_core1", Priority.MEDIUM)); + + createFile(path.resolve("h123a/abcd-efgh/dump_core1"), Instant.parse("2020-04-21T07:13:00Z")); + createFile(path.resolve("h123a/56ad-af42/dump_vespa-distributor.321"), Instant.parse("2020-04-21T23:37:00Z")); + createFile(path.resolve("h123a/4324-a23d/dump_core2"), Instant.parse("2020-04-22T04:56:00Z")); + createFile(path.resolve("h123a/8534-7da3/dump_vespa-proton-bin.123"), Instant.parse("2020-04-19T15:35:00Z")); + + // Also create a core for a second container: h123b + createFile(path.resolve("h123b/db1a-ab34/dump_core1"), Instant.parse("2020-04-21T07:01:00Z")); + createFile(path.resolve("h123b/7392-59ad/dump_vespa-proton-bin.342"), Instant.parse("2020-04-22T12:05:00Z")); + + assertPriorities(rule, Map.of( + "/test/path/h123a/abcd-1234/dump_core1", Priority.HIGH, + "/test/path/h123a/abcd-efgh/dump_core1", Priority.HIGH, + + // Although it is the oldest core of the day for h123a, it is the first one that starts with vespa- + "/test/path/h123a/56ad-af42/dump_vespa-distributor.321", Priority.MEDIUM, + "/test/path/h123a/4324-a23d/dump_core2", Priority.MEDIUM, + "/test/path/h123a/8534-7da3/dump_vespa-proton-bin.123", Priority.MEDIUM, + "/test/path/h123b/db1a-ab34/dump_core1", Priority.MEDIUM, + "/test/path/h123b/7392-59ad/dump_vespa-proton-bin.342", Priority.MEDIUM + )); + } + + private static void createFile(Path path, Instant instant) throws IOException { + Files.createDirectories(path.getParent()); + Files.createFile(path); + Files.setLastModifiedTime(path, FileTime.from(instant)); + } + + private static void assertPriorities(DiskCleanupRule rule, Map expected) { + Map actual = rule.prioritize().stream() + .collect(Collectors.toMap(pfa -> pfa.fileAttributes().path().toString(), PrioritizedFileAttributes::priority)); + + assertEquals(new TreeMap<>(expected), new TreeMap<>(actual)); + } +} \ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java new file mode 100644 index 00000000000..3f383a68b84 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java @@ -0,0 +1,142 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFileAttributes; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author freva + */ +public class DiskCleanupTest { + + private final TestTaskContext context = new TestTaskContext(); + private final DiskCleanupTester tester = new DiskCleanupTester(); + private final DiskCleanup diskCleanup = new DiskCleanup(); + + @Test + public void nothing_deleted() throws IOException { + assertFalse(diskCleanup.cleanup(context, List.of(), 0)); + assertFalse(diskCleanup.cleanup(context, List.of(), 10)); + + DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock(); + DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock(); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 0)); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + + tester.createFile("/path/that-should-not-be-deleted", 5); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + tester.assertAllFilesExistExcept(); + + // Create a file and let rule return it, but before cleanup is run, the file is deleted + rule1.addFile(tester.createFile("/path/file-does-not-exist", 1), Priority.HIGHEST); + Files.delete(tester.path("/path/file-does-not-exist")); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + } + + @Test + public void delete_test() throws IOException { + tester.createFile("/opt/vespa/var/db/do-not-delete-1.db", 1); + tester.createFile("/opt/vespa/var/db/do-not-delete-2.db", 1); + tester.createFile("/opt/vespa/var/zookeeper/do-not-delete-3", 1); + tester.createFile("/opt/vespa/var/index/something-important", 1); + + DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock() + .addFile(tester.createFile("/opt/vespa/logs/vespa-1.log", 10), Priority.MEDIUM) + .addFile(tester.createFile("/opt/vespa/logs/vespa-2.log", 8), Priority.HIGH) + .addFile(tester.createFile("/opt/vespa/logs/vespa-3.log", 13), Priority.HIGHEST) + .addFile(tester.createFile("/opt/vespa/logs/vespa-4.log", 10), Priority.HIGHEST); + DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock() + .addFile(tester.createFile("/opt/vespa/var/crash/core1", 105), Priority.LOW) + .addFile(tester.createFile("/opt/vespa/var/crash/vespa-proton-bin.core-232", 190), Priority.HIGH) + .addFile(tester.createFile("/opt/vespa/var/crash/core3", 54), Priority.MEDIUM) + .addFile(tester.createFile("/opt/vespa/var/crash/core4", 300), Priority.HIGH); + + // 2 files with HIGHEST priority, tie broken by the largest size which is won by "vespa-3.log", since + // it is >= 10 bytes, no more files are deleted + assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log"); + + // Called with the same arguments, but vespa-3.log is still missing... + assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log"); + + assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 500)); + tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log", // from before + // 300 + 190 + 8 + 54 + "/opt/vespa/var/crash/core4", "/opt/vespa/var/crash/vespa-proton-bin.core-232", "/opt/vespa/logs/vespa-2.log", "/opt/vespa/var/crash/core3"); + } + + @Test + public void bytes_to_display_count_test() { + assertEquals("-1 bytes", DiskCleanup.bytesToDisplayCount(-1)); + assertEquals("123 bytes", DiskCleanup.bytesToDisplayCount(123)); + assertEquals("1 kB", DiskCleanup.bytesToDisplayCount(1_000)); + assertEquals("15 MB", DiskCleanup.bytesToDisplayCount(15_000_000)); + assertEquals("123 GB", DiskCleanup.bytesToDisplayCount(123_456_789_012L)); + assertEquals("987 TB", DiskCleanup.bytesToDisplayCount(987_654_321_098_765L)); + assertEquals("2 PB", DiskCleanup.bytesToDisplayCount(2_000_000_000_000_000L)); + assertEquals("9 EB", DiskCleanup.bytesToDisplayCount(Long.MAX_VALUE)); + + } + + private static class DiskCleanupRuleMock implements DiskCleanupRule { + private final ArrayList pfa = new ArrayList<>(); + + private DiskCleanupRuleMock addFile(Path path, Priority priority) throws IOException { + PosixFileAttributes attributes = Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes(); + pfa.add(new PrioritizedFileAttributes(new FileAttributes(path, attributes), priority)); + return this; + } + + @Override + public Collection prioritize() { + return Collections.unmodifiableList(pfa); + } + } + + private static class DiskCleanupTester { + private final FileSystem fileSystem = TestFileSystem.create(); + private final Set files = new HashSet<>(); + + private Path path(String path) { + return fileSystem.getPath(path); + } + + private Path createFile(String pathStr, int size) throws IOException { + Path path = path(pathStr); + Files.createDirectories(path.getParent()); + Files.write(path, new byte[size]); + files.add(path.toString()); + return path; + } + + private void assertAllFilesExistExcept(String... deletedPaths) { + Set actual = FileFinder.files(path("/")).stream().map(fa -> fa.path().toString()).collect(Collectors.toSet()); + Set expected = new HashSet<>(files); + expected.removeAll(Set.of(deletedPaths)); + assertEquals(expected, actual); + } + } +} \ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java new file mode 100644 index 00000000000..ae2d3af4f21 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java @@ -0,0 +1,55 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import org.junit.Test; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * @author freva + */ +public class LinearCleanupRuleTest { + + @Test + public void basic() { + assertRule(Map.of(), Priority.LOWEST, Priority.HIGHEST); + + assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.LOW, 1.0, Priority.LOW), Priority.LOW, Priority.LOW); + assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.MEDIUM, 1.0, Priority.MEDIUM), Priority.LOW, Priority.MEDIUM); + + assertRule(Map.of( + -5.0, Priority.LOW, + 0.0, Priority.LOW, + 0.2, Priority.LOW, + 0.35, Priority.MEDIUM, + 0.65, Priority.MEDIUM, + 0.8, Priority.HIGH, + 1.0, Priority.HIGH, + 5.0, Priority.HIGH), + Priority.LOW, Priority.HIGH); + } + + @Test(expected = IllegalArgumentException.class) + public void fail_if_high_priority_lower_than_low() { + assertRule(Map.of(), Priority.HIGHEST, Priority.LOWEST); + } + + private static void assertRule(Map expectedPriorities, Priority low, Priority high) { + Map fileAttributesByScore = expectedPriorities.keySet().stream() + .collect(Collectors.toMap(score -> mock(FileAttributes.class), score -> score)); + LinearCleanupRule rule = new LinearCleanupRule( + () -> List.copyOf(fileAttributesByScore.keySet()), fileAttributesByScore::get, low, high); + + Map actualPriorities = rule.prioritize().stream() + .collect(Collectors.toMap(pfa -> fileAttributesByScore.get(pfa.fileAttributes()), PrioritizedFileAttributes::priority)); + assertEquals(expectedPriorities, actualPriorities); + } +} \ No newline at end of file -- cgit v1.2.3 From f5b0f594949a63836a662efaba8900c2f1e904d6 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 22 Apr 2020 14:02:14 +0200 Subject: Use DiskCleanup to clean disk when full --- .../node/admin/maintenance/StorageMaintainer.java | 88 +++++++++++++--------- .../hosted/node/admin/nodeagent/NodeAgentImpl.java | 9 +-- .../admin/maintenance/StorageMaintainerTest.java | 45 ++++++++++- .../node/admin/nodeagent/NodeAgentImplTest.java | 19 +---- 4 files changed, 98 insertions(+), 63 deletions(-) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index c1897c3c416..7a7a0a8e2bc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -3,10 +3,17 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.yahoo.config.provision.NodeType; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.dockerapi.Container; +import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; +import com.yahoo.vespa.hosted.node.admin.maintenance.disk.CoredumpCleanupRule; +import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup; +import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule; +import com.yahoo.vespa.hosted.node.admin.maintenance.disk.LinearCleanupRule; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; @@ -17,18 +24,20 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.logging.Logger; -import java.util.regex.Pattern; -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameMatches; -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.olderThan; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; import static com.yahoo.yolean.Exceptions.uncheck; /** @@ -42,34 +51,34 @@ public class StorageMaintainer { private final Terminal terminal; private final CoredumpHandler coredumpHandler; private final Path archiveContainerStoragePath; + private final DiskCleanup diskCleanup; private final Clock clock; // We cache disk usage to avoid doing expensive disk operations so often - private final Cache diskUsage = CacheBuilder.newBuilder() + private final Cache diskUsage = CacheBuilder.newBuilder() .maximumSize(100) .expireAfterWrite(5, TimeUnit.MINUTES) .build(); public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath) { - this(terminal, coredumpHandler, archiveContainerStoragePath, Clock.systemUTC()); + this(terminal, coredumpHandler, archiveContainerStoragePath, new DiskCleanup(), Clock.systemUTC()); } - public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath, Clock clock) { + public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath, DiskCleanup diskCleanup, Clock clock) { this.terminal = terminal; this.coredumpHandler = coredumpHandler; this.archiveContainerStoragePath = archiveContainerStoragePath; + this.diskCleanup = diskCleanup; this.clock = clock; } public Optional getDiskUsageFor(NodeAgentContext context) { try { - Path path = context.pathOnHostFromPathInNode("/"); - - Long cachedDiskUsage = diskUsage.getIfPresent(path); + Long cachedDiskUsage = diskUsage.getIfPresent(context.containerName()); if (cachedDiskUsage != null) return Optional.of(cachedDiskUsage); - long diskUsageBytes = getDiskUsedInBytes(context, path); - diskUsage.put(path, diskUsageBytes); + long diskUsageBytes = getDiskUsedInBytes(context, context.pathOnHostFromPathInNode("/")); + diskUsage.put(context.containerName(), diskUsageBytes); return Optional.of(diskUsageBytes); } catch (Exception e) { context.log(logger, LogLevel.WARNING, "Failed to get disk usage", e); @@ -95,37 +104,44 @@ public class StorageMaintainer { return 1024 * Long.parseLong(results[0]); } + public boolean cleanDiskIfFull(NodeAgentContext context) { + double totalBytes = context.node().diskGb() * 1_000_000_000L; + // Delete enough bytes to get below 80% disk usage, but only if we are already using more than 90% disk + long bytesToRemove = getDiskUsageFor(context) + .map(diskUsage -> (long) (diskUsage - 0.8 * totalBytes)) + .filter(bytes -> bytes > totalBytes * 0.1) + .orElse(0L); + + if (bytesToRemove > 0 && diskCleanup.cleanup(context, createCleanupRules(context), bytesToRemove)) { + diskUsage.invalidate(context.containerName()); + return true; + } + return false; + } - /** Deletes old log files for vespa and nginx */ - public void removeOldFilesFromNode(NodeAgentContext context) { - Path[] logPaths = { - context.pathInNodeUnderVespaHome("logs/nginx"), - context.pathInNodeUnderVespaHome("logs/vespa") - }; + private List createCleanupRules(NodeAgentContext context) { + Instant start = clock.instant(); + double oneMonthSeconds = Duration.ofDays(30).getSeconds(); + Function monthNormalizer = instant -> Duration.between(instant, start).getSeconds() / oneMonthSeconds; + Function pathOnHostUnderContainerVespaHome = path -> + context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome(path)); + List rules = new ArrayList<>(); - for (Path pathToClean : logPaths) { - Path path = context.pathOnHostFromPathInNode(pathToClean); - FileFinder.files(path) - .match(olderThan(Duration.ofDays(3)).and(nameMatches(Pattern.compile(".*\\.log.+")))) - .maxDepth(1) - .deleteRecursively(context); - } + rules.add(CoredumpCleanupRule.forContainer(pathOnHostUnderContainerVespaHome.apply("var/crash"))); - FileFinder.files(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("logs/vespa/qrs"))) - .match(olderThan(Duration.ofDays(3))) - .deleteRecursively(context); + if (context.node().membership().map(m -> m.type().isContainer()).orElse(false)) + rules.add(new LinearCleanupRule(() -> FileFinder.files(pathOnHostUnderContainerVespaHome.apply("logs/vespa/qrs")).list(), + fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.HIGHEST)); - FileFinder.files(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("logs/vespa/logarchive"))) - .match(olderThan(Duration.ofDays(31))) - .deleteRecursively(context); + if (context.nodeType() == NodeType.tenant && context.node().membership().map(m -> m.type().isAdmin()).orElse(false)) + rules.add(new LinearCleanupRule(() -> FileFinder.files(pathOnHostUnderContainerVespaHome.apply("logs/vespa/logarchive")).list(), + fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.HIGHEST)); - FileFinder.directories(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("var/db/vespa/filedistribution"))) - .match(olderThan(Duration.ofDays(31))) - .deleteRecursively(context); + if (context.nodeType() == NodeType.proxy) + rules.add(new LinearCleanupRule(() -> FileFinder.files(pathOnHostUnderContainerVespaHome.apply("logs/nginx")).list(), + fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.MEDIUM)); - FileFinder.directories(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("var/db/vespa/download"))) - .match(olderThan(Duration.ofDays(31))) - .deleteRecursively(context); + return rules; } /** Checks if container has any new coredumps, reports and archives them if so */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 9c8b0ec2b86..1a79edca203 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -48,9 +48,6 @@ import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.Containe */ public class NodeAgentImpl implements NodeAgent { - // This is used as a definition of 1 GB when comparing flavor specs in node-repo - private static final long BYTES_IN_GB = 1_000_000_000L; - // Container is started with uncapped CPU and is kept that way until the first successful health check + this duration // Subtract 1 second to avoid warmup coming in lockstep with tick time and always end up using an extra tick when there are just a few ms left private static final Duration DEFAULT_WARM_UP_DURATION = Duration.ofSeconds(90).minus(Duration.ofSeconds(1)); @@ -445,13 +442,9 @@ public class NodeAgentImpl implements NodeAgent { stopServicesIfNeeded(context); break; case active: + storageMaintainer.cleanDiskIfFull(context); storageMaintainer.handleCoreDumpsForContainer(context, container); - storageMaintainer.getDiskUsageFor(context) - .map(diskUsage -> (double) diskUsage / BYTES_IN_GB / node.diskGb()) - .filter(diskUtil -> diskUtil >= 0.8) - .ifPresent(diskUtil -> storageMaintainer.removeOldFilesFromNode(context)); - if (downloadImageIfNeeded(node, container)) { context.log(logger, "Waiting for image to download " + context.node().wantedDockerImage().get().asString()); return; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 50a8da63b96..47f24ba67c3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -1,7 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance; +import com.yahoo.config.provision.NodeResources; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; @@ -25,6 +28,11 @@ import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; /** * @author dybis @@ -85,7 +93,7 @@ public class StorageMaintainerTest { // Archive container-1 ManualClock clock = new ManualClock(Instant.ofEpochSecond(1234567890)); - StorageMaintainer storageMaintainer = new StorageMaintainer(null, null, pathToArchiveDir, clock); + StorageMaintainer storageMaintainer = new StorageMaintainer(null, null, pathToArchiveDir, null, clock); storageMaintainer.archiveNodeStorage(context1); clock.advance(Duration.ofSeconds(3)); @@ -140,4 +148,39 @@ public class StorageMaintainerTest { return context; } } + + public static class CleanupTests { + + private final TestTerminal terminal = new TestTerminal(); + private final DiskCleanup diskCleanup = mock(DiskCleanup.class); + private final ManualClock clock = new ManualClock(); + private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, null, null, diskCleanup, clock); + private final FileSystem fileSystem = TestFileSystem.create(); + private final NodeAgentContext context = new NodeAgentContextImpl + .Builder(NodeSpec.Builder.testSpec("h123a.domain.tld").resources(new NodeResources(1, 1, 1, 1)).build()) + .pathToContainerStorageFromFileSystem(fileSystem).build(); + + @Test + public void not_run_if_not_enough_used() throws IOException { + Files.createDirectories(context.pathOnHostFromPathInNode("/")); + mockDiskUsage(500L); + + storageMaintainer.cleanDiskIfFull(context); + verifyNoInteractions(diskCleanup); + } + + @Test + public void deletes_correct_amount() throws IOException { + Files.createDirectories(context.pathOnHostFromPathInNode("/")); + mockDiskUsage(950_000L); + + storageMaintainer.cleanDiskIfFull(context); + // Allocated size: 1 GB, usage: 950_000 kiB (972.8 MB). Wanted usage: 80% => 800 MB + verify(diskCleanup).cleanup(eq(context), any(), eq(172_800_000L)); + } + + private void mockDiskUsage(long kBytes) { + terminal.expectCommand("du -xsk /home/docker/h123a 2>&1", 0, kBytes + "\t/path"); + } + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index b895300d7b1..dc1c0466d39 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -78,14 +78,12 @@ public class NodeAgentImplTest { NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(187500000000L)); nodeAgent.doConverge(context); verify(dockerOperations, never()).removeContainer(eq(context), any()); verify(orchestrator, never()).suspend(any(String.class)); verify(dockerOperations, never()).pullImageAsyncIfNeeded(any()); - verify(storageMaintainer, never()).removeOldFilesFromNode(eq(context)); final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository); // TODO: Verify this isn't run unless 1st time @@ -104,11 +102,10 @@ public class NodeAgentImplTest { NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(217432719360L)); nodeAgent.doConverge(context); - verify(storageMaintainer, times(1)).removeOldFilesFromNode(eq(context)); + verify(storageMaintainer, times(1)).cleanDiskIfFull(eq(context)); } @Test @@ -122,7 +119,6 @@ public class NodeAgentImplTest { NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(187500000000L)); nodeAgent.doConverge(context); inOrder.verify(dockerOperations, never()).startServices(eq(context)); @@ -158,7 +154,6 @@ public class NodeAgentImplTest { when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(context); @@ -191,7 +186,6 @@ public class NodeAgentImplTest { when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(context); @@ -214,7 +208,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); - when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); InOrder inOrder = inOrder(orchestrator, dockerOperations); @@ -262,7 +255,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); - when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(firstContext); NodeAgentContext secondContext = createContext(specBuilder.memoryGb(20).build()); @@ -323,7 +315,6 @@ public class NodeAgentImplTest { when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); doThrow(new ConvergenceException("Connection refused")).doNothing() .when(healthChecker).verifyHealth(eq(context)); @@ -495,7 +486,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, false); when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(context); @@ -517,7 +507,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); final InOrder inOrder = inOrder(orchestrator, dockerOperations, nodeRepository); doThrow(new RuntimeException("Failed 1st time")) @@ -553,7 +542,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = spy(makeNodeAgent(null, false)); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); doThrow(new DockerException("Failed to set up network")).doNothing().when(dockerOperations).startContainer(eq(context)); try { @@ -591,7 +579,6 @@ public class NodeAgentImplTest { when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(context); @@ -629,8 +616,6 @@ public class NodeAgentImplTest { NodeAgentContext context = createContext(specBuilder.build()); NodeAgentImpl nodeAgent = makeNodeAgent(null, false, Duration.ofSeconds(30)); - when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); - InOrder inOrder = inOrder(orchestrator, dockerOperations); ConvergenceException healthCheckException = new ConvergenceException("Not yet up"); @@ -685,8 +670,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); - when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); - InOrder inOrder = inOrder(orchestrator, dockerOperations); nodeAgent.doConverge(context); -- cgit v1.2.3 From c7cfd6a9de2745d0cfd71b2eb2d3dc58b7326e35 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 22 Apr 2020 14:22:03 +0000 Subject: Change interface to avoid danerous temporary. --- searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp index 9ca49107079..fd722d0676f 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp @@ -261,7 +261,7 @@ DocumentFieldNode::visitMembers(vespalib::ObjectVisitor &visitor) const class String2ResultNode : public ResultNode { public: - String2ResultNode(const vespalib::string & s) : _s(s) { } + String2ResultNode(vespalib::stringref s) : _s(s) { } int64_t onGetInteger(size_t index) const override { (void) index; return strtoul(_s.c_str(), NULL, 0); } double onGetFloat(size_t index) const override { (void) index; return vespalib::locale::c::strtod(_s.c_str(), NULL); } ConstBufferRef onGetString(size_t index, BufferRef buf) const override { (void) index; (void) buf; return ConstBufferRef(_s.c_str(), _s.size()); } @@ -269,7 +269,7 @@ private: String2ResultNode * clone() const override { return new String2ResultNode(_s); } void set(const ResultNode&) override; size_t hash() const override { return 0; } - const vespalib::string & _s; + vespalib::string _s; }; void String2ResultNode::set(const ResultNode&) -- cgit v1.2.3 From e9ca4ef4898c15b1c07df5a002646e35be5bd9b3 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 22 Apr 2020 18:56:56 +0200 Subject: Some libraries print "0x0" for a null void ptr, --- storage/src/tests/bucketdb/judyarraytest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/src/tests/bucketdb/judyarraytest.cpp b/storage/src/tests/bucketdb/judyarraytest.cpp index 94d61107fcf..72967e8b4c0 100644 --- a/storage/src/tests/bucketdb/judyarraytest.cpp +++ b/storage/src/tests/bucketdb/judyarraytest.cpp @@ -62,7 +62,7 @@ TEST(JudyArrayTest, iterating) { JudyArray::ConstIterator cit = array.begin(); EXPECT_THAT(cit.toString(), MatchesRegex("^ConstIterator\\(Key: 3, Valp: 0x[0-9a-f]{1,16}, Val: 2\\)$")); JudyArray::Iterator it = array.end(); - EXPECT_THAT(it.toString(), MatchesRegex("^Iterator\\(Key: 0, Valp: 0\\)$")); + EXPECT_THAT(it.toString(), MatchesRegex("^Iterator\\(Key: 0, Valp: (0x)?0\\)$")); } } -- cgit v1.2.3 From 53f8942c5309ceb56c6dbe820df9dbefef0baf00 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 22 Apr 2020 19:02:25 +0200 Subject: Don't mix iterators from different containers. --- .../storage/distributor/maintenance/simplebucketprioritydatabase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp index 56f43db1ac5..7b75091e252 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp @@ -83,7 +83,7 @@ SimpleBucketPriorityDatabase::SimpleConstIteratorImpl::increment() { while (!atEnd()) { step(); - if (!currentPriorityAtEnd()) { + if (atEnd() || !currentPriorityAtEnd()) { break; } } -- cgit v1.2.3 From 61d3b63e937e4fdf9e5e12e36218cb6acd3e3c2b Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 22 Apr 2020 22:41:27 +0200 Subject: Create DiskSize --- .../configserver/noderepository/NodeSpec.java | 5 ++ .../node/admin/maintenance/StorageMaintainer.java | 25 +++++---- .../node/admin/maintenance/disk/DiskCleanup.java | 13 ++--- .../hosted/node/admin/task/util/file/DiskSize.java | 63 ++++++++++++++++++++++ .../admin/maintenance/StorageMaintainerTest.java | 9 ++-- .../admin/maintenance/disk/DiskCleanupTest.java | 15 +----- .../node/admin/task/util/file/DiskSizeTest.java | 25 +++++++++ 7 files changed, 117 insertions(+), 38 deletions(-) create mode 100644 node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java create mode 100644 node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index 85c35369a22..76616a3a8f5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; import java.time.Instant; import java.util.EnumSet; @@ -220,6 +221,10 @@ public class NodeSpec { return resources.memoryGb(); } + public DiskSize diskSize() { + return DiskSize.of(resources.diskGb(), DiskSize.Unit.GB); + } + public double diskGb() { return resources.diskGb(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 7a7a0a8e2bc..9c718955c0e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.disk.LinearCleanupRule; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; @@ -55,7 +56,7 @@ public class StorageMaintainer { private final Clock clock; // We cache disk usage to avoid doing expensive disk operations so often - private final Cache diskUsage = CacheBuilder.newBuilder() + private final Cache diskUsage = CacheBuilder.newBuilder() .maximumSize(100) .expireAfterWrite(5, TimeUnit.MINUTES) .build(); @@ -72,12 +73,17 @@ public class StorageMaintainer { this.clock = clock; } + // TODO: Remove, use diskUsageFor() instead public Optional getDiskUsageFor(NodeAgentContext context) { + return diskUsageFor(context).map(DiskSize::bytes); + } + + public Optional diskUsageFor(NodeAgentContext context) { try { - Long cachedDiskUsage = diskUsage.getIfPresent(context.containerName()); + DiskSize cachedDiskUsage = diskUsage.getIfPresent(context.containerName()); if (cachedDiskUsage != null) return Optional.of(cachedDiskUsage); - long diskUsageBytes = getDiskUsedInBytes(context, context.pathOnHostFromPathInNode("/")); + DiskSize diskUsageBytes = getDiskUsed(context, context.pathOnHostFromPathInNode("/")); diskUsage.put(context.containerName(), diskUsageBytes); return Optional.of(diskUsageBytes); } catch (Exception e) { @@ -86,9 +92,8 @@ public class StorageMaintainer { } } - // Public for testing - long getDiskUsedInBytes(TaskContext context, Path path) { - if (!Files.exists(path)) return 0; + DiskSize getDiskUsed(TaskContext context, Path path) { + if (!Files.exists(path)) return DiskSize.ZERO; String output = terminal.newCommandLine(context) .add("du", "-xsk", path.toString()) @@ -101,14 +106,14 @@ public class StorageMaintainer { throw new ConvergenceException("Result from disk usage command not as expected: " + output); } - return 1024 * Long.parseLong(results[0]); + return DiskSize.of(Long.parseLong(results[0]), DiskSize.Unit.kiB); } public boolean cleanDiskIfFull(NodeAgentContext context) { - double totalBytes = context.node().diskGb() * 1_000_000_000L; + double totalBytes = context.node().diskSize().bytes(); // Delete enough bytes to get below 80% disk usage, but only if we are already using more than 90% disk - long bytesToRemove = getDiskUsageFor(context) - .map(diskUsage -> (long) (diskUsage - 0.8 * totalBytes)) + long bytesToRemove = diskUsageFor(context) + .map(diskUsage -> (long) (diskUsage.bytes() - 0.8 * totalBytes)) .filter(bytes -> bytes > totalBytes * 0.1) .orElse(0L); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java index a9c1f8a9600..c9a35d09bff 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.disk; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; import java.nio.file.Files; import java.nio.file.Path; @@ -43,8 +44,8 @@ public class DiskCleanup { }); } finally { - String wantedDeleteSize = bytesToDisplayCount(bytesToRemove); - String deletedSize = bytesToDisplayCount(bytesToRemove - btr[0]); + String wantedDeleteSize = DiskSize.of(bytesToRemove).asString(); + String deletedSize = DiskSize.of(bytesToRemove - btr[0]).asString(); if (deletedPaths.size() > 20) { context.log(logger, "Deleted %d files (%s) because disk was getting full", deletedPaths.size(), deletedSize); } else if (deletedPaths.size() > 0) { @@ -56,12 +57,4 @@ public class DiskCleanup { return !deletedPaths.isEmpty(); } - - static String bytesToDisplayCount(long bytes) { - if (bytes < 1000) return bytes + " bytes"; - - int unit = -1; - for (; bytes >= 1000; unit++) bytes /= 1000; - return bytes + " " + UNITS[unit] + "B"; - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java new file mode 100644 index 00000000000..e6d8835ef9c --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java @@ -0,0 +1,63 @@ +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.util.Objects; + +/** + * @author freva + */ +public class DiskSize { + + public static final DiskSize ZERO = DiskSize.of(0); + private static final char[] UNITS = "kMGTPE".toCharArray(); + + public enum Unit { + kB(1000), kiB(1 << 10), + MB(1_000_000), MiB(1 << 20), + GB(1_000_000_000), GiB(1 << 30), + PB(1_000_000_000_000L), PiB(1L << 40); + + private final long size; + Unit(long size) { this.size = size; } + } + + private final long bytes; + private DiskSize(long bytes) { this.bytes = bytes; } + + public long bytes() { return bytes; } + public long as(Unit unit) { return bytes / unit.size; } + public double asDouble(Unit unit) { return (double) bytes / unit.size; } + + public DiskSize add(DiskSize other) { return new DiskSize(bytes + other.bytes); } + + public static DiskSize of(long bytes) { return new DiskSize(bytes); } + public static DiskSize of(double bytes, Unit unit) { return new DiskSize((long) (bytes * unit.size)); } + public static DiskSize of(long bytes, Unit unit) { return new DiskSize(bytes * unit.size); } + + public String asString() { return asString(0); } + public String asString(int decimals) { + if (bytes < 1000) return bytes + " bytes"; + + int unit = -1; + double remaining = bytes; + for (; remaining >= 1000; unit++) remaining /= 1000; + return String.format("%." + decimals + "f %sB", remaining, UNITS[unit]); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DiskSize size = (DiskSize) o; + return bytes == size.bytes; + } + + @Override + public int hashCode() { + return Objects.hash(bytes); + } + + @Override + public String toString() { + return asString(); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 47f24ba67c3..c17d0017269 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.After; @@ -52,17 +53,17 @@ public class StorageMaintainerTest { Files.createDirectories(context.pathOnHostFromPathInNode("/")); terminal.expectCommand("du -xsk /home/docker/host-1 2>&1", 0, "321\t/home/docker/host-1/"); - assertEquals(Optional.of(328_704L), storageMaintainer.getDiskUsageFor(context)); + assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); // Value should still be cached, no new execution against the terminal - assertEquals(Optional.of(328_704L), storageMaintainer.getDiskUsageFor(context)); + assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); } @Test public void testNonExistingDiskUsed() { StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, null, null); - long usedBytes = storageMaintainer.getDiskUsedInBytes(null, Paths.get("/fake/path")); - assertEquals(0L, usedBytes); + DiskSize size = storageMaintainer.getDiskUsed(null, Paths.get("/fake/path")); + assertEquals(DiskSize.ZERO, size); } @After diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java index 3f383a68b84..7065b261b1b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java @@ -88,19 +88,6 @@ public class DiskCleanupTest { "/opt/vespa/var/crash/core4", "/opt/vespa/var/crash/vespa-proton-bin.core-232", "/opt/vespa/logs/vespa-2.log", "/opt/vespa/var/crash/core3"); } - @Test - public void bytes_to_display_count_test() { - assertEquals("-1 bytes", DiskCleanup.bytesToDisplayCount(-1)); - assertEquals("123 bytes", DiskCleanup.bytesToDisplayCount(123)); - assertEquals("1 kB", DiskCleanup.bytesToDisplayCount(1_000)); - assertEquals("15 MB", DiskCleanup.bytesToDisplayCount(15_000_000)); - assertEquals("123 GB", DiskCleanup.bytesToDisplayCount(123_456_789_012L)); - assertEquals("987 TB", DiskCleanup.bytesToDisplayCount(987_654_321_098_765L)); - assertEquals("2 PB", DiskCleanup.bytesToDisplayCount(2_000_000_000_000_000L)); - assertEquals("9 EB", DiskCleanup.bytesToDisplayCount(Long.MAX_VALUE)); - - } - private static class DiskCleanupRuleMock implements DiskCleanupRule { private final ArrayList pfa = new ArrayList<>(); @@ -139,4 +126,4 @@ public class DiskCleanupTest { assertEquals(expected, actual); } } -} \ No newline at end of file +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java new file mode 100644 index 00000000000..bcf3651fdfe --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java @@ -0,0 +1,25 @@ +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class DiskSizeTest { + + @Test + public void bytes_to_display_count_test() { + assertEquals("-1 bytes", DiskSize.of(-1).asString()); + assertEquals("123 bytes", DiskSize.of(123).asString()); + assertEquals("1 kB", DiskSize.of(1_000).asString()); + assertEquals("15 MB", DiskSize.of(15_000_000).asString()); + assertEquals("123 GB", DiskSize.of(123_456_789_012L).asString()); + assertEquals("988 TB", DiskSize.of(987_654_321_098_765L).asString()); + assertEquals("987.7 TB", DiskSize.of(987_654_321_098_765L).asString(1)); + assertEquals("987.65 TB", DiskSize.of(987_654_321_098_765L).asString(2)); + assertEquals("2 PB", DiskSize.of(2_000_000_000_000_000L).asString()); + assertEquals("9 EB", DiskSize.of(Long.MAX_VALUE).asString()); + } +} -- cgit v1.2.3 From 638d2407cc0e5b75c094e5496b1baea6231f2bbf Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 23 Apr 2020 00:54:05 +0200 Subject: Split suggesting and autoscaling --- .../hosted/provision/applications/Application.java | 6 +- .../hosted/provision/applications/Cluster.java | 27 +++++- .../hosted/provision/autoscale/Autoscaler.java | 35 +++++-- .../provision/autoscale/ResourceIterator.java | 18 ++-- .../maintenance/AutoscalingMaintainer.java | 22 ++--- .../maintenance/NodeRepositoryMaintenance.java | 5 + .../maintenance/ScalingSuggestionsMaintainer.java | 74 +++++++++++++++ .../persistence/ApplicationSerializer.java | 3 + .../provisioning/NodeRepositoryProvisioner.java | 2 +- .../autoscale/AutoscalingIntegrationTest.java | 7 +- .../provision/autoscale/AutoscalingTest.java | 21 ++++- .../provision/autoscale/AutoscalingTester.java | 9 ++ .../ScalingSuggestionsMaintainerTest.java | 101 +++++++++++++++++++++ .../persistence/ApplicationSerializerTest.java | 3 + .../restapi/v2/responses/maintenance.json | 3 + 15 files changed, 290 insertions(+), 46 deletions(-) create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java index e90a5f132ed..05aeae45ad9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -39,6 +39,10 @@ public class Application { public Map clusters() { return clusters; } + public Optional cluster(ClusterSpec.Id id) { + return Optional.ofNullable(clusters.get(id)); + } + public Application with(Cluster cluster) { Map clusters = new HashMap<>(this.clusters); clusters.put(cluster.id(), cluster); @@ -52,7 +56,7 @@ public class Application { public Application withClusterLimits(ClusterSpec.Id id, ClusterResources min, ClusterResources max) { Cluster cluster = clusters.get(id); if (cluster == null) - cluster = new Cluster(id, min, max, Optional.empty()); + cluster = new Cluster(id, min, max, Optional.empty(), Optional.empty()); else cluster = cluster.withLimits(min, max); return with(cluster); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 914bc6c5a48..9e0a9f368a8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -20,17 +20,19 @@ public class Cluster { private final ClusterSpec.Id id; private final ClusterResources min, max; + private final Optional suggested; private final Optional target; public Cluster(ClusterSpec.Id id, ClusterResources minResources, ClusterResources maxResources, + Optional suggestedResources, Optional targetResources) { this.id = Objects.requireNonNull(id); this.min = Objects.requireNonNull(minResources); this.max = Objects.requireNonNull(maxResources); + this.suggested = Objects.requireNonNull(suggestedResources); Objects.requireNonNull(targetResources); - if (targetResources.isPresent() && ! targetResources.get().isWithin(minResources, maxResources)) this.target = Optional.empty(); else @@ -47,20 +49,35 @@ public class Cluster { /** * Returns the computed resources (between min and max, inclusive) this cluster should - * have allocated at the moment, or empty if the system currently have no opinion on this. + * have allocated at the moment (whether or not it actually has it), + * or empty if the system currently has no target. */ public Optional targetResources() { return target; } + /** + * The suggested size of this cluster, which may or may not be within the min and max limits, + * or empty if there is currently no suggestion. + */ + public Optional suggestedResources() { return suggested; } + public Cluster withLimits(ClusterResources min, ClusterResources max) { - return new Cluster(id, min, max, target); + return new Cluster(id, min, max, suggested, target); + } + + public Cluster withSuggested(ClusterResources suggested) { + return new Cluster(id, min, max, Optional.of(suggested), target); + } + + public Cluster withoutSuggested() { + return new Cluster(id, min, max, Optional.empty(), target); } public Cluster withTarget(ClusterResources target) { - return new Cluster(id, min, max, Optional.of(target)); + return new Cluster(id, min, max, suggested, Optional.of(target)); } public Cluster withoutTarget() { - return new Cluster(id, min, max, Optional.empty()); + return new Cluster(id, min, max, suggested, Optional.empty()); } public NodeResources capAtLimits(NodeResources resources) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 447d3494fbc..14d1dca3e81 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -31,7 +31,6 @@ public class Autoscaler { - Scale group size - Consider taking spikes/variance into account - Measure observed regulation lag (startup+redistribution) and take it into account when deciding regulation observation window - - Test AutoscalingMaintainer - Scale by performance not just load+cost */ @@ -57,12 +56,28 @@ public class Autoscaler { } /** - * Autoscale a cluster + * Suggest a scaling of a cluster. This returns a better allocation (if found) + * without taking min and max limits into account. + * + * @param clusterNodes the list of all the active nodes in a cluster + * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time + */ + public Optional suggest(Cluster cluster, List clusterNodes) { + return autoscale(cluster, clusterNodes, false); + } + + /** + * Autoscale a cluster. This returns a better allocation (if found) inside the min and max limits. * * @param clusterNodes the list of all the active nodes in a cluster * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time */ public Optional autoscale(Cluster cluster, List clusterNodes) { + if (cluster.minResources().equals(cluster.maxResources())) return Optional.empty(); // Shortcut + return autoscale(cluster, clusterNodes, true); + } + + private Optional autoscale(Cluster cluster, List clusterNodes, boolean respectLimits) { if (unstable(clusterNodes)) return Optional.empty(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); @@ -76,7 +91,8 @@ public class Autoscaler { memoryLoad.get(), diskLoad.get(), currentAllocation, - cluster); + cluster, + respectLimits); if (bestAllocation.isEmpty()) return Optional.empty(); if (similar(bestAllocation.get(), currentAllocation)) return Optional.empty(); return bestAllocation; @@ -84,12 +100,14 @@ public class Autoscaler { private Optional findBestAllocation(double cpuLoad, double memoryLoad, double diskLoad, AllocatableClusterResources currentAllocation, - Cluster cluster) { + Cluster cluster, boolean respectLimits) { Optional bestAllocation = Optional.empty(); - for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation, cluster); i.hasNext(); ) { + for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation, cluster, respectLimits); + i.hasNext(); ) { Optional allocatableResources = toAllocatableResources(i.next(), currentAllocation.clusterType(), - cluster); + cluster, + respectLimits); if (allocatableResources.isEmpty()) continue; if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) bestAllocation = allocatableResources; @@ -119,9 +137,10 @@ public class Autoscaler { */ private Optional toAllocatableResources(ClusterResources resources, ClusterSpec.Type clusterType, - Cluster cluster) { + Cluster cluster, + boolean respectLimits) { NodeResources nodeResources = resources.nodeResources(); - if ( ! cluster.minResources().equals(cluster.maxResources())) // enforce application limits unless suggest mode + if (respectLimits) nodeResources = cluster.capAtLimits(nodeResources); nodeResources = nodeResourceLimits.enlargeToLegal(nodeResources, clusterType); // enforce system limits diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java index b7d5995884e..207eecc1871 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java @@ -11,8 +11,8 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster; */ public class ResourceIterator { - // Configured min and max nodes for suggestions for apps which have not activated autoscaling - private static final int minimumNodes = 3; // Since this is with redundancy it cannot be lower than 2 + // The min and max nodes to consider when not using application supplied limits + private static final int minimumNodes = 3; // Since this number includes redundancy it cannot be lower than 2 private static final int maximumNodes = 150; // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component @@ -22,6 +22,7 @@ public class ResourceIterator { // Prescribed state private final Cluster cluster; + private final boolean respectLimits; // Observed state private final AllocatableClusterResources allocation; @@ -39,10 +40,12 @@ public class ResourceIterator { public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad, AllocatableClusterResources currentAllocation, - Cluster cluster) { + Cluster cluster, + boolean respectLimits) { this.cpuLoad = cpuLoad; this.memoryLoad = memoryLoad; this.diskLoad = diskLoad; + this.respectLimits = respectLimits; // ceil: If the division does not produce a whole number we assume some node is missing groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups()); @@ -64,11 +67,6 @@ public class ResourceIterator { currentNodes -= nodeIncrement; } - /** If autoscaling is not enabled (meaning max and min resources are the same), we want to suggest */ - private boolean suggestMode() { - return cluster.minResources().equals(cluster.maxResources()); - } - public ClusterResources next() { ClusterResources next = resourcesWith(currentNodes); currentNodes += nodeIncrement; @@ -80,13 +78,13 @@ public class ResourceIterator { } private int minNodes() { - if (suggestMode()) return minimumNodes; + if ( ! respectLimits) return minimumNodes; if (singleGroupMode) return cluster.minResources().nodes(); return Math.max(cluster.minResources().nodes(), cluster.minResources().groups() * groupSize ); } private int maxNodes() { - if (suggestMode()) return maximumNodes; + if ( ! respectLimits) return maximumNodes; if (singleGroupMode) return cluster.maxResources().nodes(); return Math.min(cluster.maxResources().nodes(), cluster.maxResources().groups() * groupSize ); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index d17577af135..3b97195ad2d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -34,7 +34,6 @@ public class AutoscalingMaintainer extends Maintainer { private final Autoscaler autoscaler; private final Deployer deployer; private final Metric metric; - private final Map, Instant> lastLogged = new HashMap<>(); public AutoscalingMaintainer(NodeRepository nodeRepository, HostResourcesCalculator hostResourcesCalculator, @@ -68,17 +67,13 @@ public class AutoscalingMaintainer extends Maintainer { MaintenanceDeployment deployment) { Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)); Cluster cluster = application.clusters().get(clusterId); - if (cluster == null) return; // no information on limits for this cluster + if (cluster == null) return; + if (cluster.minResources().equals(cluster.maxResources())) return; Optional target = autoscaler.autoscale(cluster, clusterNodes); if (target.isEmpty()) return; // current resources are fine - if (cluster.minResources().equals(cluster.maxResources())) { // autoscaling is deactivated - logAutoscaling("Scaling suggestion for ", target.get(), applicationId, clusterId, clusterNodes); - } - else { - logAutoscaling("Autoscaling ", target.get(), applicationId, clusterId, clusterNodes); - autoscaleTo(target.get(), clusterId, application, deployment); - } + logAutoscaling(target.get(), applicationId, clusterId, clusterNodes); + autoscaleTo(target.get(), clusterId, application, deployment); } private void autoscaleTo(AllocatableClusterResources target, @@ -90,20 +85,15 @@ public class AutoscalingMaintainer extends Maintainer { deployment.activate(); } - private void logAutoscaling(String prefix, - AllocatableClusterResources target, + private void logAutoscaling(AllocatableClusterResources target, ApplicationId application, ClusterSpec.Id clusterId, List clusterNodes) { - Instant lastLogTime = lastLogged.get(new Pair<>(application, clusterId)); - if (lastLogTime != null && lastLogTime.isAfter(nodeRepository().clock().instant().minus(Duration.ofHours(1)))) return; - int currentGroups = (int)clusterNodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); - log.info(prefix + application + " " + clusterType + " " + clusterId + ":" + + log.info("Autoscaling " + application + " " + clusterType + " " + clusterId + ":" + "\nfrom " + toString(clusterNodes.size(), currentGroups, clusterNodes.get(0).flavor().resources()) + "\nto " + toString(target.nodes(), target.groups(), target.advertisedResources())); - lastLogged.put(new Pair<>(application, clusterId), nodeRepository().clock().instant()); } private String toString(int nodes, int groups, NodeResources resources) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 054c273dc99..9e954e0a1dd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -52,6 +52,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Rebalancer rebalancer; private final NodeMetricsDbMaintainer nodeMetricsDbMaintainer; private final AutoscalingMaintainer autoscalingMaintainer; + private final ScalingSuggestionsMaintainer scalingSuggestionsMaintainer; @SuppressWarnings("unused") @Inject @@ -92,6 +93,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval); autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, deployer, metric, defaults.autoscalingInterval); + scalingSuggestionsMaintainer = new ScalingSuggestionsMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, defaults.scalingSuggestionsInterval); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintainButThrowOnException(); @@ -118,6 +120,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebalancer.deconstruct(); nodeMetricsDbMaintainer.deconstruct(); autoscalingMaintainer.deconstruct(); + scalingSuggestionsMaintainer.deconstruct(); } private static Optional throttlePolicyFromEnv() { @@ -160,6 +163,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration rebalancerInterval; private final Duration nodeMetricsCollectionInterval; private final Duration autoscalingInterval; + private final Duration scalingSuggestionsInterval; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -182,6 +186,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebalancerInterval = Duration.ofMinutes(40); nodeMetricsCollectionInterval = Duration.ofMinutes(1); autoscalingInterval = Duration.ofMinutes(5); + scalingSuggestionsInterval = Duration.ofMinutes(31); if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java new file mode 100644 index 00000000000..f80f2d11753 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -0,0 +1,74 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Applications; +import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; +import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Maintainer computing scaling suggestions for all clusters + * + * @author bratseth + */ +public class ScalingSuggestionsMaintainer extends Maintainer { + + private final Autoscaler autoscaler; + + public ScalingSuggestionsMaintainer(NodeRepository nodeRepository, + HostResourcesCalculator hostResourcesCalculator, + NodeMetricsDb metricsDb, + Duration interval) { + super(nodeRepository, interval); + this.autoscaler = new Autoscaler(hostResourcesCalculator, metricsDb, nodeRepository); + } + + @Override + protected void maintain() { + if ( ! nodeRepository().zone().environment().isProduction()) return; + + activeNodesByApplication().forEach((applicationId, nodes) -> suggest(applicationId, nodes)); + } + + private void suggest(ApplicationId application, List applicationNodes) { + nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> + suggest(application, clusterId, clusterNodes)); + } + + private void suggest(ApplicationId applicationId, + ClusterSpec.Id clusterId, + List clusterNodes) { + Applications applications = nodeRepository().applications(); + Application application = applications.get(applicationId).orElse(new Application(applicationId)); + Cluster cluster = application.clusters().get(clusterId); + if (cluster == null) return; + Optional target = autoscaler.suggest(cluster, clusterNodes); + if (target.isEmpty()) return; + ClusterResources suggestion = target.get().toAdvertisedClusterResources(); + + try (Mutex lock = nodeRepository().lock(applicationId)) { + applications.get(applicationId).ifPresent(a -> a.cluster(clusterId).ifPresent(c -> + applications.put(a.with(c.withSuggested(suggestion)), lock))); + } + } + + private Map> nodesByCluster(List applicationNodes) { + return applicationNodes.stream().collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster().id())); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index f514cc20fe8..b1453a9729a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -37,6 +37,7 @@ public class ApplicationSerializer { private static final String clustersKey = "clusters"; private static final String minResourcesKey = "min"; private static final String maxResourcesKey = "max"; + private static final String suggestedResourcesKey = "suggested"; private static final String targetResourcesKey = "target"; private static final String nodesKey = "nodes"; private static final String groupsKey = "groups"; @@ -81,6 +82,7 @@ public class ApplicationSerializer { private static void toSlime(Cluster cluster, Cursor clusterObject) { toSlime(cluster.minResources(), clusterObject.setObject(minResourcesKey)); toSlime(cluster.maxResources(), clusterObject.setObject(maxResourcesKey)); + cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject(suggestedResourcesKey))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject(targetResourcesKey))); } @@ -88,6 +90,7 @@ public class ApplicationSerializer { return new Cluster(ClusterSpec.Id.from(id), clusterResourcesFromSlime(clusterObject.field(minResourcesKey)), clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)), + optionalClusterResourcesFromSlime(clusterObject.field(suggestedResourcesKey)), optionalClusterResourcesFromSlime(clusterObject.field(targetResourcesKey))); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 103543917bc..5f1139fec2d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -170,7 +170,7 @@ public class NodeRepositoryProvisioner implements Provisioner { // the non-numeric settings of the current min limit with the current numeric settings NodeResources nodeResources = nodes.get(0).allocation().get().requestedResources() .with(requested.minResources().nodeResources().diskSpeed()) - .with(requested.maxResources().nodeResources().storageType()); + .with(requested.minResources().nodeResources().storageType()); var currentResources = new ClusterResources(nodes.size(), (int)groups, nodeResources); if ( ! currentResources.isWithin(requested.minResources(), requested.maxResources())) return Optional.empty(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java index 7c6c4e2336b..b4f6adc2d26 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -21,6 +21,9 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +/** + * @author bratseth + */ public class AutoscalingIntegrationTest { @Test @@ -55,8 +58,8 @@ public class AutoscalingIntegrationTest { Application application = tester.nodeRepository().applications().get(application1).orElse(new Application(application1)) .withClusterLimits(cluster1.id(), min, max); tester.nodeRepository().applications().put(application, tester.nodeRepository().lock(application1)); - var scaledResources = autoscaler.autoscale(application.clusters().get(cluster1.id()), - tester.nodeRepository().getNodes(application1)); + var scaledResources = autoscaler.suggest(application.clusters().get(cluster1.id()), + tester.nodeRepository().getNodes(application1)); assertTrue(scaledResources.isPresent()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 497a2a31ce5..4dcbcd10111 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -187,9 +187,24 @@ public class AutoscalingTest { tester.autoscale(application1, cluster1.id(), min, max)); } - /** This condition ensures we get recommendation suggestions when deactivated */ @Test - public void testAutoscalingLimitsAreIgnoredIfMinEqualsMax() { + public void testAutoscalingLimitsWhenMinEqualsMax() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = min; + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, resources); + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); + assertTrue(tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); + } + + @Test + public void testSuggestionsIgnoresLimits() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = min; @@ -203,7 +218,7 @@ public class AutoscalingTest { tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); tester.assertResources("Scaling up since resource usage is too high", 7, 1, 2.6, 80.0, 80.0, - tester.autoscale(application1, cluster1.id(), min, max)); + tester.suggest(application1, cluster1.id(), min, max)); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index c250a0a23b0..6eb3aaea66e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -163,6 +163,15 @@ class AutoscalingTester { nodeRepository().getNodes(applicationId, Node.State.active)); } + public Optional suggest(ApplicationId applicationId, ClusterSpec.Id clusterId, + ClusterResources min, ClusterResources max) { + Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)) + .withClusterLimits(clusterId, min, max); + nodeRepository().applications().put(application, nodeRepository().lock(applicationId)); + return autoscaler.suggest(application.clusters().get(clusterId), + nodeRepository().getNodes(applicationId, Node.State.active)); + } + public AllocatableClusterResources assertResources(String message, int nodeCount, int groupCount, double approxCpu, double approxMemory, double approxDisk, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java new file mode 100644 index 00000000000..40892d80759 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -0,0 +1,101 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.vespa.hosted.provision.autoscale.Resource; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests the scaling suggestions maintainer integration. + * The specific suggestions are not tested here. + * + * @author bratseth + */ +public class ScalingSuggestionsMaintainerTest { + + @Test + public void testScalingSuggestionsMaintainer() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east3"))).flavorsConfig(flavorsConfig()).build(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = tester.clusterSpec(); + + ApplicationId app2 = tester.makeApplicationId("app2"); + ClusterSpec cluster2 = tester.clusterSpec(); + + NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(); + + tester.makeReadyNodes(20, "flt", NodeType.host, 8); + tester.deployZoneApp(); + + tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + false, true)); + tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + new ClusterResources(10, 1, new NodeResources(6.5, 5, 15, 0.1)), + false, true)); + + addMeasurements(Resource.cpu, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.memory, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.disk, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.cpu, 0.99f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.memory, 0.99f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.disk, 0.99f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + + ScalingSuggestionsMaintainer maintainer = new ScalingSuggestionsMaintainer(tester.nodeRepository(), + tester.identityHostResourcesCalculator(), + nodeMetricsDb, + Duration.ofMinutes(1)); + maintainer.maintain(); + + assertEquals("7 nodes with [vcpu: 15.3, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps]", + tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestedResources().get().toString()); + assertEquals("7 nodes with [vcpu: 16.8, memory: 5.7 Gb, disk 16.5 Gb, bandwidth: 0.1 Gbps]", + tester.nodeRepository().applications().get(app2).get().cluster(cluster2.id()).get().suggestedResources().get().toString()); + } + + public void addMeasurements(Resource resource, float value, int count, ApplicationId applicationId, + NodeRepository nodeRepository, NodeMetricsDb db) { + List nodes = nodeRepository.getNodes(applicationId, Node.State.active); + for (int i = 0; i < count; i++) { + for (Node node : nodes) + db.add(List.of(new NodeMetrics.MetricValue(node.hostname(), + resource.metricName(), + nodeRepository.clock().instant().toEpochMilli(), + value * 100))); // the metrics are in % + } + } + + private FlavorsConfig flavorsConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("flt", 30, 30, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("cpu", 40, 20, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("mem", 20, 40, 40, 3, Flavor.Type.BARE_METAL); + return b.build(); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index 1f07e26d045..c50805eebb8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -28,10 +28,12 @@ public class ApplicationSerializerTest { clusters.add(new Cluster(ClusterSpec.Id.from("c1"), new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), new ClusterResources(12, 6, new NodeResources(3, 6, 21, 24)), + Optional.empty(), Optional.empty())); clusters.add(new Cluster(ClusterSpec.Id.from("c2"), new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), new ClusterResources(14, 7, new NodeResources(3, 6, 21, 24)), + Optional.of(new ClusterResources(20, 10, new NodeResources(0.5, 4, 14, 16))), Optional.of(new ClusterResources(10, 5, new NodeResources(2, 4, 14, 16))))); Application original = new Application(ApplicationId.from("myTenant", "myApplication", "myInstance"), clusters); @@ -49,6 +51,7 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.id(), serializedCluster.id()); assertEquals(originalCluster.minResources(), serializedCluster.minResources()); assertEquals(originalCluster.maxResources(), serializedCluster.maxResources()); + assertEquals(originalCluster.suggestedResources(), serializedCluster.suggestedResources()); assertEquals(originalCluster.targetResources(), serializedCluster.targetResources()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json index ab608bac2b4..e041a7b8b54 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json @@ -53,6 +53,9 @@ }, { "name": "RetiredExpirer" + }, + { + "name":"ScalingSuggestionsMaintainer" } ], "inactive": [ -- cgit v1.2.3 From 293c968385780de950d417fe25f3b11ef04d3c65 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 23 Apr 2020 09:21:08 +0000 Subject: Extend the timeout for config-proxy to start from 1m to 3m. --- config-proxy/src/main/sh/vespa-config-ctl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index 91e4cd39bd2..af66e6d5cc2 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -130,7 +130,7 @@ case $1 in echo "Waiting for config proxy to start" fail=true - for ((sleepcount=0;$sleepcount<600;sleepcount=$sleepcount+1)) ; do + for ((sleepcount=0;$sleepcount<1800;sleepcount=$sleepcount+1)) ; do sleep 0.1 if [ -f $P_CONFIG_PROXY ] && kill -0 `cat $P_CONFIG_PROXY` && vespa-ping-configproxy -s $hname 2>/dev/null then -- cgit v1.2.3 From ecc08a021ffb8dd24e8f0e1f44f50ba7f2a210bc Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 22 Apr 2020 23:28:11 +0200 Subject: Scout attribute index config. Ignore live config if attribute types don't match. --- .../documentdbconfigscout_test.cpp | 60 ++++++++++++++++++++-- .../searchcore/proton/attribute/CMakeLists.txt | 1 + .../proton/attribute/attribute_type_matcher.cpp | 34 ++++++++++++ .../proton/attribute/attribute_type_matcher.h | 22 ++++++++ .../proton/attribute/attributemanager.cpp | 22 ++------ .../proton/attribute/attributesconfigscout.cpp | 6 ++- 6 files changed, 121 insertions(+), 24 deletions(-) create mode 100644 searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.cpp create mode 100644 searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.h diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp index f76743b9480..2ae28936476 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp @@ -98,11 +98,32 @@ assertFastSearchAndMoreAttribute(const AttributesConfig::Attribute &attribute, return true; } +bool +assertTensorAttribute(const AttributesConfig::Attribute &attribute, + const vespalib::string &name, const vespalib::string &spec, int max_links_per_node) +{ + if (!EXPECT_EQUAL(attribute.name, name)) { + return false; + } + if (!EXPECT_EQUAL((int)attribute.datatype, (int)AttributesConfig::Attribute::Datatype::TENSOR)) { + return false; + } + if (!EXPECT_EQUAL(attribute.tensortype, spec)) { + return false; + } + if (!EXPECT_TRUE(attribute.index.hnsw.enabled)) { + return false; + } + if (!EXPECT_EQUAL(attribute.index.hnsw.maxlinkspernode, max_links_per_node)) { + return false; + } + return true; +} bool assertAttributes(const AttributesConfig::AttributeVector &attributes) { - if (!EXPECT_EQUAL(4u, attributes.size())) { + if (!EXPECT_EQUAL(6u, attributes.size())) { return false; } if (!assertDefaultAttribute(attributes[0], "a1")) { @@ -117,6 +138,12 @@ assertAttributes(const AttributesConfig::AttributeVector &attributes) if (!assertDefaultAttribute(attributes[3], "a4")) { return false; } + if (!assertTensorAttribute(attributes[4], "tensor1", "tensor(x[100])", 16)) { + return false; + } + if (!assertTensorAttribute(attributes[5], "tensor2", "tensor(x[100])", 16)) { + return false; + } return true; } @@ -124,7 +151,7 @@ assertAttributes(const AttributesConfig::AttributeVector &attributes) bool assertLiveAttributes(const AttributesConfig::AttributeVector &attributes) { - if (!EXPECT_EQUAL(5u, attributes.size())) { + if (!EXPECT_EQUAL(7u, attributes.size())) { return false; } if (!assertFastSearchAttribute(attributes[0], "a0")) { @@ -142,6 +169,12 @@ assertLiveAttributes(const AttributesConfig::AttributeVector &attributes) if (!assertFastSearchAttribute(attributes[4], "a4")) { return false; } + if (!assertTensorAttribute(attributes[5], "tensor1", "tensor(x[100])", 32)) { + return false; + } + if (!assertTensorAttribute(attributes[6], "tensor2", "tensor(x[200])", 32)) { + return false; + } return true; } @@ -149,7 +182,7 @@ assertLiveAttributes(const AttributesConfig::AttributeVector &attributes) bool assertScoutedAttributes(const AttributesConfig::AttributeVector &attributes) { - if (!EXPECT_EQUAL(4u, attributes.size())) { + if (!EXPECT_EQUAL(6u, attributes.size())) { return false; } if (!assertFastSearchAndMoreAttribute(attributes[0], "a1")) { @@ -164,6 +197,12 @@ assertScoutedAttributes(const AttributesConfig::AttributeVector &attributes) if (!assertDefaultAttribute(attributes[3], "a4")) { return false; } + if (!assertTensorAttribute(attributes[4], "tensor1", "tensor(x[100])", 32)) { + return false; + } + if (!assertTensorAttribute(attributes[5], "tensor2", "tensor(x[100])", 16)) { + return false; + } return true; } @@ -199,6 +238,17 @@ setupFastSearchAndMoreAttribute(const vespalib::string name) return attribute; } +AttributesConfig::Attribute +setupTensorAttribute(const vespalib::string &name, const vespalib::string &spec, int max_links_per_node) +{ + AttributesConfig::Attribute attribute; + attribute.name = name; + attribute.datatype = AttributesConfig::Attribute::Datatype::TENSOR; + attribute.tensortype = spec; + attribute.index.hnsw.enabled = true; + attribute.index.hnsw.maxlinkspernode = max_links_per_node; + return attribute; +} void setupDefaultAttributes(AttributesConfigBuilder::AttributeVector &attributes) @@ -207,6 +257,8 @@ setupDefaultAttributes(AttributesConfigBuilder::AttributeVector &attributes) attributes.push_back(setupDefaultAttribute("a2")); attributes.push_back(setupDefaultAttribute("a3")); attributes.push_back(setupDefaultAttribute("a4")); + attributes.push_back(setupTensorAttribute("tensor1", "tensor(x[100])", 16)); + attributes.push_back(setupTensorAttribute("tensor2", "tensor(x[100])", 16)); } @@ -221,6 +273,8 @@ setupLiveAttributes(AttributesConfigBuilder::AttributeVector &attributes) attributes.back().collectiontype = AttributesConfig::Attribute::Collectiontype::ARRAY; attributes.push_back(setupFastSearchAttribute("a4")); attributes.back().createifnonexistent = true; + attributes.push_back(setupTensorAttribute("tensor1", "tensor(x[100])", 32)); + attributes.push_back(setupTensorAttribute("tensor2", "tensor(x[200])", 32)); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt index f7a6ffe7189..550d0a1c4c4 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_add_library(searchcore_attribute STATIC attribute_manager_initializer.cpp attribute_populator.cpp attribute_spec.cpp + attribute_type_matcher.cpp attribute_usage_filter.cpp attribute_usage_sampler_context.cpp attribute_usage_sampler_functor.cpp diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.cpp new file mode 100644 index 00000000000..c99a6ae9e6e --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.cpp @@ -0,0 +1,34 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_type_matcher.h" +#include + +using search::attribute::BasicType; + +namespace proton +{ + +bool +AttributeTypeMatcher::operator()(const search::attribute::Config &oldConfig, const search::attribute::Config &newConfig) const +{ + if ((oldConfig.basicType() != newConfig.basicType()) || + (oldConfig.collectionType() != newConfig.collectionType())) { + return false; + } + if (newConfig.basicType().type() == BasicType::Type::TENSOR) { + if (oldConfig.tensorType() != newConfig.tensorType()) { + return false; + } + } + if (newConfig.basicType().type() == BasicType::Type::PREDICATE) { + using Params = search::attribute::PersistentPredicateParams; + const Params &oldParams = oldConfig.predicateParams(); + const Params &newParams = newConfig.predicateParams(); + if (!(oldParams == newParams)) { + return false; + } + } + return true; +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.h new file mode 100644 index 00000000000..bbe9479083b --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.h @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::attribute { class Config; } + +namespace proton +{ + +/** + * Class to check if attribute types are compatible or not. + */ +class AttributeTypeMatcher +{ +public: + AttributeTypeMatcher() = default; + ~AttributeTypeMatcher() = default; + bool operator()(const search::attribute::Config &oldConfig, + const search::attribute::Config &newConfig) const; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index 5ff40fa5360..214db2c2d13 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -4,6 +4,7 @@ #include "attribute_directory.h" #include "attributedisklayout.h" #include "attributemanager.h" +#include "attribute_type_matcher.h" #include "imported_attributes_context.h" #include "imported_attributes_repo.h" #include "sequential_attributes_initializer.h" @@ -41,25 +42,8 @@ namespace { bool matchingTypes(const AttributeVector::SP &av, const search::attribute::Config &newConfig) { if (av) { - const auto &oldConfig = av->getConfig(); - if ((oldConfig.basicType() != newConfig.basicType()) || - (oldConfig.collectionType() != newConfig.collectionType())) { - return false; - } - if (newConfig.basicType().type() == BasicType::TENSOR) { - if (oldConfig.tensorType() != newConfig.tensorType()) { - return false; - } - } - if (newConfig.basicType().type() == BasicType::PREDICATE) { - using Params = search::attribute::PersistentPredicateParams; - const Params &oldParams = oldConfig.predicateParams(); - const Params &newParams = newConfig.predicateParams(); - if (!(oldParams == newParams)) { - return false; - } - } - return true; + AttributeTypeMatcher matching_types; + return matching_types(av->getConfig(), newConfig); } else { return false; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp index 2f6cf9e6208..986571d07e1 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attributesconfigscout.h" +#include "attribute_type_matcher.h" #include using search::attribute::ConfigConverter; @@ -29,6 +30,7 @@ AttributesConfigScout::adjust(AttributesConfig::Attribute &attr, attr.huge = liveAttr.huge; // Note: Predicate attributes only handle changes for the dense-posting-list-threshold config. attr.densepostinglistthreshold = liveAttr.densepostinglistthreshold; + attr.index = liveAttr.index; } @@ -41,8 +43,8 @@ AttributesConfigScout::adjust(AttributesConfig::Attribute &attr) const auto &liveAttr = _live.attribute[it->second]; search::attribute::Config liveCfg = ConfigConverter::convert(liveAttr); - if (cfg.basicType() == liveCfg.basicType() && - cfg.collectionType() == liveCfg.collectionType()) { + AttributeTypeMatcher matching_types; + if (matching_types(cfg, liveCfg)) { adjust(attr, liveAttr); } } -- cgit v1.2.3 From ef417b03f65c7c78b3f496a34b26a88a2e015c6e Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Wed, 15 Apr 2020 11:40:04 +0200 Subject: Rename lockConfig -> lock --- .../vespa/hosted/provision/NodeRepository.java | 5 ++-- .../provision/maintenance/LoadBalancerExpirer.java | 2 +- .../persistence/CuratorDatabaseClient.java | 27 ++++++++++++++-------- .../provisioning/LoadBalancerProvisioner.java | 8 +++---- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 6044bce7610..8b019d54d49 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -778,11 +778,10 @@ public class NodeRepository extends AbstractComponent { public Zone zone() { return zone; } /** Create a lock which provides exclusive rights to making changes to the given application */ - // TODO(mpolden): Make this delegate to CuratorDatabaseClient#lockConfig instead - public Mutex lock(ApplicationId application) { return db.lock(application); } + public Mutex lock(ApplicationId application) { return db.legacyLock(application); } /** Create a lock with a timeout which provides exclusive rights to making changes to the given application */ - public Mutex lock(ApplicationId application, Duration timeout) { return db.lock(application, timeout); } + public Mutex lock(ApplicationId application, Duration timeout) { return db.legacyLock(application, timeout); } /** Create a lock which provides exclusive rights to modifying unallocated nodes */ public Mutex lockUnallocated() { return db.lockInactive(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index d7a93288d2d..94cca64794d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -120,7 +120,7 @@ public class LoadBalancerExpirer extends Maintainer { /** Apply operation to all load balancers that exist in given state, while holding lock */ private void withLoadBalancersIn(LoadBalancer.State state, Consumer operation) { for (var id : db.readLoadBalancerIds()) { - try (var lock = db.lockConfig(id.application())) { + try (var lock = db.lock(id.application())) { var loadBalancer = db.readLoadBalancer(id); if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop if (loadBalancer.get().state() != state) continue; // Wrong state diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index af4e054579e..a7824d07d49 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -323,7 +323,8 @@ public class CuratorDatabaseClient { } /** Creates and returns the path to the lock for this application */ - private Path lockPath(ApplicationId application) { + // TODO(mpolden): Remove when all config servers take the new lock + private Path legacyLockPath(ApplicationId application) { Path lockPath = CuratorDatabaseClient.lockPath .append(application.tenant().value()) @@ -333,8 +334,8 @@ public class CuratorDatabaseClient { return lockPath; } - /** Creates and returns the path to the config server lock for this application */ - private Path configLockPath(ApplicationId application) { + /** Creates and returns the lock path for this application */ + private Path lockPath(ApplicationId application) { // This must match the lock path used by com.yahoo.vespa.config.server.application.TenantApplications Path lockPath = configLockPath.append(application.tenant().value()).append(application.serializedForm()); curatorDatabase.create(lockPath); @@ -362,14 +363,16 @@ public class CuratorDatabaseClient { } /** Acquires the single cluster-global, reentrant lock for active nodes of this application */ - public Lock lock(ApplicationId application) { - return lock(application, defaultLockTimeout); + // TODO(mpolden): Remove when all config servers take the new lock + public Lock legacyLock(ApplicationId application) { + return legacyLock(application, defaultLockTimeout); } /** Acquires the single cluster-global, reentrant lock with the specified timeout for active nodes of this application */ - public Lock lock(ApplicationId application, Duration timeout) { + // TODO(mpolden): Remove when all config servers take the new lock + public Lock legacyLock(ApplicationId application, Duration timeout) { try { - return lock(lockPath(application), timeout); + return lock(legacyLockPath(application), timeout); } catch (UncheckedTimeoutException e) { throw new ApplicationLockException(e); @@ -377,7 +380,7 @@ public class CuratorDatabaseClient { } /** - * Acquires the single cluster-global, re-entrant config lock for given application. Note that this is the same lock + * Acquires the single cluster-global, re-entrant lock for given application. Note that this is the same lock * that configserver itself takes when modifying applications. * * This lock must be taken when writes to paths owned by this class may happen on both the configserver and @@ -390,14 +393,18 @@ public class CuratorDatabaseClient { * transaction. The config server then commits (writes) the transaction which may include operations that modify * data in paths owned by this class. */ - public Lock lockConfig(ApplicationId application) { + public Lock lock(ApplicationId application, Duration timeout) { try { - return lock(configLockPath(application), defaultLockTimeout); + return lock(lockPath(application), timeout); } catch (UncheckedTimeoutException e) { throw new ApplicationLockException(e); } } + public Lock lock(ApplicationId application) { + return lock(application, defaultLockTimeout); + } + // Applications ----------------------------------------------------------- public List readApplicationIds() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index f1fc6be837e..1683f60bbbe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -52,7 +52,7 @@ public class LoadBalancerProvisioner { this.service = service; // Read and write all load balancers to make sure they are stored in the latest version of the serialization format for (var id : db.readLoadBalancerIds()) { - try (var lock = db.lockConfig(id.application())) { + try (var lock = db.lock(id.application())) { var loadBalancer = db.readLoadBalancer(id); loadBalancer.ifPresent(db::writeLoadBalancer); } @@ -73,7 +73,7 @@ public class LoadBalancerProvisioner { if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type if (application.instance().isTester()) return; // Do not provision for tester instances - try (var lock = db.lockConfig(application)) { + try (var lock = db.lock(application)) { provision(application, effectiveId(cluster), false, lock); } } @@ -90,7 +90,7 @@ public class LoadBalancerProvisioner { */ public void activate(ApplicationId application, Set clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { - try (var lock = db.lockConfig(application)) { + try (var lock = db.lock(application)) { var containerClusters = containerClustersOf(clusters); for (var clusterId : containerClusters) { // Provision again to ensure that load balancer instance is re-configured with correct nodes @@ -108,7 +108,7 @@ public class LoadBalancerProvisioner { */ public void deactivate(ApplicationId application, NestedTransaction transaction) { try (var applicationLock = nodeRepository.lock(application)) { - try (var lock = db.lockConfig(application)) { + try (var lock = db.lock(application)) { deactivate(nodeRepository.loadBalancers(application).asList(), transaction); } } -- cgit v1.2.3 From c1580e721443ce5ba22a044714e623399524f8d0 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Thu, 23 Apr 2020 12:23:05 +0200 Subject: Unwrap reference wrappers to avoid extra indirections via invalid memory. --- eval/src/vespa/eval/eval/make_tensor_function.cpp | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/eval/src/vespa/eval/eval/make_tensor_function.cpp b/eval/src/vespa/eval/eval/make_tensor_function.cpp index f503532c1f9..3a73a3b8784 100644 --- a/eval/src/vespa/eval/eval/make_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/make_tensor_function.cpp @@ -59,37 +59,37 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void make_reduce(const Node &, Aggr aggr, const std::vector &dimensions) { assert(stack.size() >= 1); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::reduce(a, aggr, dimensions, stash); } void make_map(const Node &, map_fun_t function) { assert(stack.size() >= 1); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::map(a, function, stash); } void make_join(const Node &, join_fun_t function) { assert(stack.size() >= 2); - const auto &b = stack.back(); + const auto &b = stack.back().get(); stack.pop_back(); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::join(a, b, function, stash); } void make_merge(const Node &, join_fun_t function) { assert(stack.size() >= 2); - const auto &b = stack.back(); + const auto &b = stack.back().get(); stack.pop_back(); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::merge(a, b, function, stash); } void make_concat(const Node &, const vespalib::string &dimension) { assert(stack.size() >= 2); - const auto &b = stack.back(); + const auto &b = stack.back().get(); stack.pop_back(); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::concat(a, b, dimension, stash); } @@ -156,17 +156,17 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void make_rename(const Node &, const std::vector &from, const std::vector &to) { assert(stack.size() >= 1); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::rename(a, from, to, stash); } void make_if(const Node &) { assert(stack.size() >= 3); - const auto &c = stack.back(); + const auto &c = stack.back().get(); stack.pop_back(); - const auto &b = stack.back(); + const auto &b = stack.back().get(); stack.pop_back(); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::if_node(a, b, c, stash); } -- cgit v1.2.3 From 4d1842587174b5f3344ac3115b1f822d8c62f0b3 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Wed, 15 Apr 2020 12:48:49 +0200 Subject: Inline unnecessary method --- .../persistence/CuratorDatabaseClient.java | 100 ++++++++++----------- 1 file changed, 48 insertions(+), 52 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index a7824d07d49..1a238d5c3b6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -76,7 +76,7 @@ public class CuratorDatabaseClient { private final NodeSerializer nodeSerializer; private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); - private final CuratorDatabase curatorDatabase; + private final CuratorDatabase db; private final Clock clock; private final Zone zone; private final CuratorCounter provisionIndexCounter; @@ -84,27 +84,27 @@ public class CuratorDatabaseClient { public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, boolean useCache) { this.nodeSerializer = new NodeSerializer(flavors); this.zone = zone; - this.curatorDatabase = new CuratorDatabase(curator, root, useCache); + this.db = new CuratorDatabase(curator, root, useCache); this.clock = clock; this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter").getAbsolute()); initZK(); } public List cluster() { - return curatorDatabase.cluster(); + return db.cluster(); } private void initZK() { - curatorDatabase.create(root); + db.create(root); for (Node.State state : Node.State.values()) - curatorDatabase.create(toPath(state)); - curatorDatabase.create(applicationsPath); - curatorDatabase.create(inactiveJobsPath); - curatorDatabase.create(infrastructureVersionsPath); - curatorDatabase.create(osVersionsPath); - curatorDatabase.create(dockerImagesPath); - curatorDatabase.create(firmwareCheckPath); - curatorDatabase.create(loadBalancersPath); + db.create(toPath(state)); + db.create(applicationsPath); + db.create(inactiveJobsPath); + db.create(infrastructureVersionsPath); + db.create(osVersionsPath); + db.create(dockerImagesPath); + db.create(firmwareCheckPath); + db.create(loadBalancersPath); provisionIndexCounter.initialize(100); } @@ -113,7 +113,7 @@ public class CuratorDatabaseClient { */ public List addNodesInState(List nodes, Node.State expectedState) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); for (Node node : nodes) { if (node.state() != expectedState) throw new IllegalArgumentException(node + " is not in the " + expectedState + " state"); @@ -139,7 +139,7 @@ public class CuratorDatabaseClient { for (Node node : nodes) { Path path = toPath(node.state(), node.hostname()); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.delete(path.getAbsolute())); } @@ -209,7 +209,7 @@ public class CuratorDatabaseClient { List writtenNodes = new ArrayList<>(nodes.size()); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); for (Node node : nodes) { Node newNode = new Node(node.id(), node.ipConfig(), node.hostname(), node.parentHostname(), node.flavor(), @@ -269,7 +269,7 @@ public class CuratorDatabaseClient { List nodes = new ArrayList<>(); if (states.length == 0) states = Node.State.values(); - CuratorDatabase.Session session = curatorDatabase.getSession(); + CuratorDatabase.Session session = db.getSession(); for (Node.State state : states) { for (String hostname : session.getChildren(toPath(state))) { Optional node = readNode(session, hostname, state); @@ -309,7 +309,7 @@ public class CuratorDatabaseClient { * If no states are given this returns the node if it is present in any state. */ public Optional readNode(String hostname, Node.State ... states) { - return readNode(curatorDatabase.getSession(), hostname, states); + return readNode(db.getSession(), hostname, states); } private Path toPath(Node.State nodeState) { return root.append(toDir(nodeState)); } @@ -330,7 +330,7 @@ public class CuratorDatabaseClient { .append(application.tenant().value()) .append(application.application().value()) .append(application.instance().value()); - curatorDatabase.create(lockPath); + db.create(lockPath); return lockPath; } @@ -338,7 +338,7 @@ public class CuratorDatabaseClient { private Path lockPath(ApplicationId application) { // This must match the lock path used by com.yahoo.vespa.config.server.application.TenantApplications Path lockPath = configLockPath.append(application.tenant().value()).append(application.serializedForm()); - curatorDatabase.create(lockPath); + db.create(lockPath); return lockPath; } @@ -359,7 +359,7 @@ public class CuratorDatabaseClient { /** Acquires the single cluster-global, reentrant lock for all non-active nodes */ public Lock lockInactive() { - return lock(lockPath.append("unallocatedLock"), defaultLockTimeout); + return db.lock(lockPath.append("unallocatedLock"), defaultLockTimeout); } /** Acquires the single cluster-global, reentrant lock for active nodes of this application */ @@ -372,7 +372,7 @@ public class CuratorDatabaseClient { // TODO(mpolden): Remove when all config servers take the new lock public Lock legacyLock(ApplicationId application, Duration timeout) { try { - return lock(legacyLockPath(application), timeout); + return db.lock(legacyLockPath(application), timeout); } catch (UncheckedTimeoutException e) { throw new ApplicationLockException(e); @@ -395,7 +395,7 @@ public class CuratorDatabaseClient { */ public Lock lock(ApplicationId application, Duration timeout) { try { - return lock(lockPath(application), timeout); + return db.lock(lockPath(application), timeout); } catch (UncheckedTimeoutException e) { throw new ApplicationLockException(e); } @@ -408,9 +408,9 @@ public class CuratorDatabaseClient { // Applications ----------------------------------------------------------- public List readApplicationIds() { - return curatorDatabase.getChildren(applicationsPath).stream() - .map(path -> ApplicationId.fromSerializedForm(path)) - .collect(Collectors.toList()); + return db.getChildren(applicationsPath).stream() + .map(path -> ApplicationId.fromSerializedForm(path)) + .collect(Collectors.toList()); } public Optional readApplication(ApplicationId id) { @@ -418,15 +418,15 @@ public class CuratorDatabaseClient { } public void writeApplication(Application application, NestedTransaction transaction) { - curatorDatabase.newCuratorTransactionIn(transaction) - .add(createOrSet(applicationPath(application.id()), + db.newCuratorTransactionIn(transaction) + .add(createOrSet(applicationPath(application.id()), ApplicationSerializer.toJson(application))); } public void deleteApplication(ApplicationId application, NestedTransaction transaction) { - if (curatorDatabase.exists(applicationPath(application))) - curatorDatabase.newCuratorTransactionIn(transaction) - .add(CuratorOperations.delete(applicationPath(application).getAbsolute())); + if (db.exists(applicationPath(application))) + db.newCuratorTransactionIn(transaction) + .add(CuratorOperations.delete(applicationPath(application).getAbsolute())); } private Path applicationPath(ApplicationId id) { @@ -436,7 +436,7 @@ public class CuratorDatabaseClient { // Maintenance jobs ----------------------------------------------------------- public Lock lockMaintenanceJob(String jobName) { - return lock(lockPath.append("maintenanceJobLocks").append(jobName), defaultLockTimeout); + return db.lock(lockPath.append("maintenanceJobLocks").append(jobName), defaultLockTimeout); } public Set readInactiveJobs() { @@ -452,14 +452,14 @@ public class CuratorDatabaseClient { public void writeInactiveJobs(Set inactiveJobs) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(inactiveJobsPath.getAbsolute(), stringSetSerializer.toJson(inactiveJobs))); transaction.commit(); } public Lock lockInactiveJobs() { - return lock(lockPath.append("inactiveJobsLock"), defaultLockTimeout); + return db.lock(lockPath.append("inactiveJobsLock"), defaultLockTimeout); } // Infrastructure versions ----------------------------------------------------------- @@ -470,14 +470,14 @@ public class CuratorDatabaseClient { public void writeInfrastructureVersions(Map infrastructureVersions) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(infrastructureVersionsPath.getAbsolute(), NodeTypeVersionsSerializer.toJson(infrastructureVersions))); transaction.commit(); } public Lock lockInfrastructureVersions() { - return lock(lockPath.append("infrastructureVersionsLock"), defaultLockTimeout); + return db.lock(lockPath.append("infrastructureVersionsLock"), defaultLockTimeout); } // OS versions ----------------------------------------------------------- @@ -488,14 +488,14 @@ public class CuratorDatabaseClient { public void writeOsVersions(Map versions) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(osVersionsPath.getAbsolute(), OsVersionsSerializer.toJson(versions))); transaction.commit(); } public Lock lockOsVersions() { - return lock(lockPath.append("osVersionsLock"), defaultLockTimeout); + return db.lock(lockPath.append("osVersionsLock"), defaultLockTimeout); } // Docker images ----------------------------------------------------------- @@ -506,14 +506,14 @@ public class CuratorDatabaseClient { public void writeDockerImages(Map dockerImages) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(dockerImagesPath.getAbsolute(), NodeTypeDockerImagesSerializer.toJson(dockerImages))); transaction.commit(); } public Lock lockDockerImages() { - return lock(lockPath.append("dockerImagesLock"), defaultLockTimeout); + return db.lock(lockPath.append("dockerImagesLock"), defaultLockTimeout); } // Firmware checks ----------------------------------------------------------- @@ -523,7 +523,7 @@ public class CuratorDatabaseClient { byte[] data = after.map(instant -> Long.toString(instant.toEpochMilli()).getBytes()) .orElse(new byte[0]); NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(firmwareCheckPath.getAbsolute(), data)); transaction.commit(); } @@ -559,7 +559,7 @@ public class CuratorDatabaseClient { } public void writeLoadBalancers(Collection loadBalancers, NestedTransaction transaction) { - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); loadBalancers.forEach(loadBalancer -> { curatorTransaction.add(createOrSet(loadBalancerPath(loadBalancer.id()), LoadBalancerSerializer.toJson(loadBalancer))); @@ -568,7 +568,7 @@ public class CuratorDatabaseClient { public void removeLoadBalancer(LoadBalancerId loadBalancer) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer).getAbsolute())); transaction.commit(); } @@ -578,10 +578,10 @@ public class CuratorDatabaseClient { } private List readLoadBalancerIds(Predicate predicate) { - return curatorDatabase.getChildren(loadBalancersPath).stream() - .map(LoadBalancerId::fromSerializedForm) - .filter(predicate) - .collect(Collectors.toUnmodifiableList()); + return db.getChildren(loadBalancersPath).stream() + .map(LoadBalancerId::fromSerializedForm) + .filter(predicate) + .collect(Collectors.toUnmodifiableList()); } /** Returns a given number of unique provision indexes */ @@ -595,16 +595,12 @@ public class CuratorDatabaseClient { .collect(Collectors.toList()); } - private Lock lock(Path path, Duration timeout) { - return curatorDatabase.lock(path, timeout); - } - private Optional read(Path path, Function mapper) { - return curatorDatabase.getData(path).filter(data -> data.length > 0).map(mapper); + return db.getData(path).filter(data -> data.length > 0).map(mapper); } private Transaction.Operation createOrSet(Path path, byte[] data) { - return curatorDatabase.exists(path) ? CuratorOperations.setData(path.getAbsolute(), data) + return db.exists(path) ? CuratorOperations.setData(path.getAbsolute(), data) : CuratorOperations.create(path.getAbsolute(), data); } -- cgit v1.2.3 From 57b0a1358051414db57820d7a964b2aa13aade80 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 23 Apr 2020 14:22:10 +0200 Subject: Allocate new nodes instead of unretire+resize When we have retired nodes of the wrong size and then set wantToRetire on existing nodes, allocate new nodes instead of unretiring old nodes and resizing them. --- .../provision/provisioning/NodeAllocation.java | 41 ++++++++++------------ .../hosted/provision/provisioning/NodeSpec.java | 12 +++++++ .../provisioning/InPlaceResizeProvisionTest.java | 23 +++++++++++- 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 652fd5c6861..896a5f78d93 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -52,11 +52,11 @@ class NodeAllocation { /** The nodes this has accepted so far */ private final Set nodes = new LinkedHashSet<>(); - /** The number of nodes in the accepted nodes which are of the requested flavor */ - private int acceptedOfRequestedFlavor = 0; + /** The number of already allocated nodes accepted and not retired */ + private int accepted = 0; - /** The number of nodes in the accepted nodes which are of the requested flavor and not already retired */ - private int acceptedNonretiredOfRequestedFlavor = 0; + /** The number of already allocated nodes accepted and not retired and not needing resize */ + private int acceptedWithoutResize = 0; /** The number of nodes rejected because of clashing parentHostname */ private int rejectedWithClashingParentHost = 0; @@ -118,14 +118,14 @@ class NodeAllocation { if (offered.status().wantToRetire()) wantToRetireNode = true; if (requestedNodes.isExclusive() && ! hostsOnly(application.tenant(), offered.parentHostname())) wantToRetireNode = true; - if (( ! saturatedByNonretired() && hasCompatibleFlavor(node)) || acceptToRetire(node)) + if ((! saturated() && hasCompatibleFlavor(node)) || acceptToRetire(node)) accepted.add(acceptNode(node, wantToRetireNode, node.isResizable)); } else { accepted.add(acceptNode(node, false, false)); } } - else if ( ! saturated() && hasCompatibleFlavor(node)) { + else if (! saturated() && hasCompatibleFlavor(node)) { if ( violatesParentHostPolicy(this.nodes, offered)) { ++rejectedWithClashingParentHost; continue; @@ -226,22 +226,22 @@ class NodeAllocation { return requestedNodes.isCompatible(node.node.flavor(), flavors) || node.isResizable; } - private Node acceptNode(PrioritizableNode prioritizableNode, boolean wantToRetire, boolean resize) { + private Node acceptNode(PrioritizableNode prioritizableNode, boolean wantToRetire, boolean resizeable) { Node node = prioritizableNode.node; if (node.allocation().isPresent()) // Record the currently requested resources node = node.with(node.allocation().get().withRequestedResources(requestedNodes.resources().orElse(node.flavor().resources()))); if (! wantToRetire) { - if (resize && ! ( node.allocation().isPresent() && node.allocation().get().membership().retired())) + accepted++; + if (node.allocation().isEmpty() || ! requestedNodes.needsResize(node)) + acceptedWithoutResize++; + + if (resizeable && ! ( node.allocation().isPresent() && node.allocation().get().membership().retired())) node = resize(node); if (node.state() != Node.State.active) // reactivated node - make sure its not retired node = node.unretire(); - - acceptedOfRequestedFlavor++; - if ( ! (node.allocation().isPresent() && node.allocation().get().membership().retired())) - acceptedNonretiredOfRequestedFlavor++; } else { ++wasRetiredJustNow; // Retire nodes which are of an unwanted flavor, retired flavor or have an overlapping parent host @@ -272,29 +272,24 @@ class NodeAllocation { /** Returns true if no more nodes are needed in this list */ private boolean saturated() { - return requestedNodes.saturatedBy(acceptedOfRequestedFlavor); - } - - /** Returns true if no more nodes are needed in this list to not make changes to the retired set */ - private boolean saturatedByNonretired() { - return requestedNodes.saturatedBy(acceptedNonretiredOfRequestedFlavor); + return requestedNodes.saturatedBy(acceptedWithoutResize); } /** Returns true if the content of this list is sufficient to meet the request */ boolean fulfilled() { - return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor); + return requestedNodes.fulfilledBy(accepted); } boolean wouldBeFulfilledWithRetiredNodes() { - return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + wasRetiredJustNow); + return requestedNodes.fulfilledBy(accepted + wasRetiredJustNow); } boolean wouldBeFulfilledWithClashingParentHost() { - return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + rejectedWithClashingParentHost); + return requestedNodes.fulfilledBy(accepted + rejectedWithClashingParentHost); } boolean wouldBeFulfilledWithoutExclusivity() { - return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + rejectedDueToExclusivity); + return requestedNodes.fulfilledBy(accepted + rejectedDueToExclusivity); } /** @@ -307,7 +302,7 @@ class NodeAllocation { Optional getFulfilledDockerDeficit() { return Optional.of(requestedNodes) .filter(NodeSpec.CountNodeSpec.class::isInstance) - .map(spec -> new FlavorCount(spec.resources().get(), spec.fulfilledDeficitCount(acceptedOfRequestedFlavor))) + .map(spec -> new FlavorCount(spec.resources().get(), spec.fulfilledDeficitCount(accepted))) .filter(flavorCount -> flavorCount.getCount() > 0); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index 45fc1e6934c..a8abdc3f38a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; import java.util.Objects; import java.util.Optional; @@ -56,6 +57,9 @@ public interface NodeSpec { /** Returns the resources requested by this or empty if none are explicitly requested */ Optional resources(); + /** Returns whether the given node must be resized to match this spec */ + boolean needsResize(Node node); + /** * Returns true if a node with given current resources and current spare host resources can be resized * in-place to resources in this spec. @@ -133,6 +137,11 @@ public interface NodeSpec { return new CountNodeSpec(count/divisor, requestedNodeResources, exclusive, canFail); } + @Override + public boolean needsResize(Node node) { + return ! node.flavor().resources().compatibleWith(requestedNodeResources); + } + @Override public boolean canResize(NodeResources currentNodeResources, NodeResources currentSpareHostResources, boolean hasTopologyChange, int currentClusterSize) { @@ -198,6 +207,9 @@ public interface NodeSpec { @Override public Optional resources() { return Optional.empty(); } + @Override + public boolean needsResize(Node node) { return false; } + @Override public String toString() { return "request for all nodes of type '" + type + "'"; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java index 7e9c3ef09dc..3a9d87b9db6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.Agent; import org.junit.Test; import java.util.Collection; @@ -23,6 +24,7 @@ import java.util.stream.Collectors; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.StorageType.local; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -159,7 +161,7 @@ public class InPlaceResizeProvisionTest { /** In this scenario there should be no resizing */ @Test public void increase_size_decrease_resources() { - addParentHosts(12, largeResources.with(fast)); + addParentHosts(14, largeResources.with(fast)); NodeResources resources = new NodeResources(4, 8, 16, 1); NodeResources halvedResources = new NodeResources(2, 4, 8, 1); @@ -176,6 +178,25 @@ public class InPlaceResizeProvisionTest { new PrepareHelper(tester, app).prepare(container1, 8, 1, halvedResources).activate(); assertSizeAndResources(listCluster(container1).retired(), 4, resources); assertSizeAndResources(listCluster(container1).not().retired(), 8, halvedResources); + + // Failing one of the new nodes should cause another new node to be allocated rather than + // unretiring one of the existing nodes, to avoid resizing during unretiring + Node nodeToFail = listCluster(container1).not().retired().asList().get(0); + tester.nodeRepository().fail(nodeToFail.hostname(), Agent.system, "testing"); + new PrepareHelper(tester, app).prepare(container1, 8, 1, halvedResources).activate(); + assertFalse(listCluster(container1).stream().anyMatch(n -> n.equals(nodeToFail))); + assertSizeAndResources(listCluster(container1).retired(), 4, resources); + assertSizeAndResources(listCluster(container1).not().retired(), 8, halvedResources); + + // ... same with setting a node to want to retire + System.out.println("marking wantToRetire"); + Node nodeToWantoToRetire = listCluster(container1).not().retired().asList().get(0); + tester.nodeRepository().write(nodeToWantoToRetire.with(nodeToWantoToRetire.status().withWantToRetire(true)), + tester.nodeRepository().lock(nodeToWantoToRetire)); + new PrepareHelper(tester, app).prepare(container1, 8, 1, halvedResources).activate(); + assertTrue(listCluster(container1).retired().stream().anyMatch(n -> n.equals(nodeToWantoToRetire))); + assertEquals(5, listCluster(container1).retired().size()); + assertSizeAndResources(listCluster(container1).not().retired(), 8, halvedResources); } @Test(expected = OutOfCapacityException.class) -- cgit v1.2.3 From d389b0e2519a139dc9103a22702be175698262d8 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Thu, 23 Apr 2020 12:35:58 +0000 Subject: Add support for aggregating over B-tree keys This complements the existing support for aggregating over values. Let aggregate calculator specify whether it expects to be invoked with keys or values. In the current implementation there is some code duplication that could have been removed by using e.g. tag dispatch, but this is a pragmatic choice intended to guarantee these changes do not introduce any performance regressions for existing code using value aggregation. Can be refactored later once we have a functional baseline with this for both keys and values. --- vespalib/src/tests/btree/btreeaggregation_test.cpp | 82 +++++++++++++++++++--- .../src/vespa/vespalib/btree/btreeaggregator.hpp | 6 +- .../src/vespa/vespalib/btree/btreeinserter.hpp | 6 +- .../src/vespa/vespalib/btree/btreeiterator.hpp | 16 +++-- vespalib/src/vespa/vespalib/btree/btreeremover.hpp | 14 ++-- vespalib/src/vespa/vespalib/btree/btreestore.hpp | 6 +- vespalib/src/vespa/vespalib/btree/minmaxaggrcalc.h | 1 + vespalib/src/vespa/vespalib/btree/noaggrcalc.h | 2 + 8 files changed, 109 insertions(+), 24 deletions(-) diff --git a/vespalib/src/tests/btree/btreeaggregation_test.cpp b/vespalib/src/tests/btree/btreeaggregation_test.cpp index 0ce3d2c7d04..92d8955ab93 100644 --- a/vespalib/src/tests/btree/btreeaggregation_test.cpp +++ b/vespalib/src/tests/btree/btreeaggregation_test.cpp @@ -1,7 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include -LOG_SETUP("btreeaggregation_test"); -#include + #include #include #include @@ -17,6 +15,7 @@ LOG_SETUP("btreeaggregation_test"); #include #include #include +#include #include #include @@ -24,11 +23,13 @@ LOG_SETUP("btreeaggregation_test"); #include #include +#include +LOG_SETUP("btreeaggregation_test"); + using vespalib::GenerationHandler; using search::datastore::EntryRef; -namespace search { -namespace btree { +namespace search::btree { namespace { @@ -99,6 +100,11 @@ typedef std::less MyComp; #define UNWRAP(key) (key) #endif +struct KeyMinMaxAggrCalc : MinMaxAggrCalc { + constexpr static bool aggregate_over_values() { return false; } + constexpr static int32_t getVal(const MyKey& key) { return key._val; } +}; + typedef BTree; class MockTree { @@ -311,10 +318,13 @@ private: size_t numEntries); void requireThatNodeInsertWorks(); + void keys_are_aggregated_correctly_on_node_insertions(); void requireThatNodeSplitInsertWorks(); + void keys_are_aggregated_correctly_when_node_split_on_insert(); void requireThatTreeInsertWorks(); void requireThatNodeStealWorks(); void requireThatNodeRemoveWorks(); + void keys_are_aggregated_correctly_on_node_removal(); void requireThatWeCanInsertAndRemoveFromTree(); void requireThatSortedTreeInsertWorks(); void requireThatCornerCaseTreeFindWorks(); @@ -409,6 +419,17 @@ Test::requireThatNodeInsertWorks() EXPECT_TRUE(assertTree("{{10:101,20:102,30:103,40:104[min=101,max=104]}}", t)); } +void Test::keys_are_aggregated_correctly_on_node_insertions() { + MyKeyAggrTree t; + t.insert(20, 102); + EXPECT_TRUE(assertTree("{{20:102[min=20,max=20]}}", t)); + t.insert(10, 101); + EXPECT_TRUE(assertTree("{{10:101,20:102[min=10,max=20]}}", t)); + t.insert(30, 103); + t.insert(40, 104); + EXPECT_TRUE(assertTree("{{10:101,20:102,30:103,40:104[min=10,max=40]}}", t)); +} + template void populateTree(Tree &t, uint32_t count, uint32_t delta) @@ -422,8 +443,9 @@ populateTree(Tree &t, uint32_t count, uint32_t delta) } } +template void -populateLeafNode(MyTree &t) +populateLeafNode(Tree &t) { populateTree(t, 4, 2); } @@ -457,6 +479,33 @@ Test::requireThatNodeSplitInsertWorks() } } +void Test::keys_are_aggregated_correctly_when_node_split_on_insert() { + { // new entry in current node + MyKeyAggrTree t; + populateLeafNode(t); + t.insert(4, 104); + EXPECT_TRUE(assertTree("{{4,7[min=1,max=7]}} -> " + "{{1:101,3:103,4:104[min=1,max=4]}," + "{5:105,7:107[min=5,max=7]}}", t)); + } + { // new entry in split node + MyKeyAggrTree t; + populateLeafNode(t); + t.insert(6, 106); + EXPECT_TRUE(assertTree("{{5,7[min=1,max=7]}} -> " + "{{1:101,3:103,5:105[min=1,max=5]}," + "{6:106,7:107[min=6,max=7]}}", t)); + } + { // new entry at end + MyKeyAggrTree t; + populateLeafNode(t); + t.insert(8, 108); + EXPECT_TRUE(assertTree("{{5,8[min=1,max=8]}} -> " + "{{1:101,3:103,5:105[min=1,max=5]}," + "{7:107,8:108[min=7,max=8]}}", t)); + } +} + void Test::requireThatTreeInsertWorks() { @@ -645,6 +694,17 @@ Test::requireThatNodeRemoveWorks() EXPECT_TRUE(assertTree("{{5:105[min=105,max=105]}}", t)); } +void Test::keys_are_aggregated_correctly_on_node_removal() { + MyKeyAggrTree t; + populateLeafNode(t); + t.remove(3); + EXPECT_TRUE(assertTree("{{1:101,5:105,7:107[min=1,max=7]}}", t)); + t.remove(1); + EXPECT_TRUE(assertTree("{{5:105,7:107[min=5,max=7]}}", t)); + t.remove(7); + EXPECT_TRUE(assertTree("{{5:105[min=5,max=5]}}", t)); +} + void generateData(std::vector & data, size_t numEntries) { @@ -960,10 +1020,8 @@ struct UpdKeyComp { void Test::requireThatUpdateOfKeyWorks() { - typedef BTree UpdKeyTree; - typedef UpdKeyTree::Iterator UpdKeyTreeIterator; + using UpdKeyTree = BTree; + using UpdKeyTreeIterator = UpdKeyTree::Iterator; GenerationHandler g; UpdKeyTree t; UpdKeyComp cmp1(0); @@ -1134,10 +1192,13 @@ Test::Main() TEST_INIT("btreeaggregation_test"); requireThatNodeInsertWorks(); + keys_are_aggregated_correctly_on_node_insertions(); requireThatNodeSplitInsertWorks(); + keys_are_aggregated_correctly_when_node_split_on_insert(); requireThatTreeInsertWorks(); requireThatNodeStealWorks(); requireThatNodeRemoveWorks(); + keys_are_aggregated_correctly_on_node_removal(); requireThatWeCanInsertAndRemoveFromTree(); requireThatSortedTreeInsertWorks(); requireThatCornerCaseTreeFindWorks(); @@ -1151,7 +1212,6 @@ Test::Main() TEST_DONE(); } -} } TEST_APPHOOK(search::btree::Test); diff --git a/vespalib/src/vespa/vespalib/btree/btreeaggregator.hpp b/vespalib/src/vespa/vespalib/btree/btreeaggregator.hpp index e1318ab5a66..34dbfd3e899 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeaggregator.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeaggregator.hpp @@ -13,7 +13,11 @@ BTreeAggregator::aggr { AggrT a; for (uint32_t i = 0, ie = node.validSlots(); i < ie; ++i) { - aggrCalc.add(a, aggrCalc.getVal(node.getData(i))); + if constexpr (AggrCalcT::aggregate_over_values()) { + aggrCalc.add(a, aggrCalc.getVal(node.getData(i))); + } else { + aggrCalc.add(a, aggrCalc.getVal(node.getKey(i))); + } } return a; } diff --git a/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp b/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp index e1518da5639..5c6b02a5a46 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp @@ -121,7 +121,11 @@ insert(BTreeNode::Ref &root, lnode->insert(idx, key, data); itr.setLeafNodeIdx(idx); if constexpr (AggrCalcT::hasAggregated()) { - aggrCalc.add(lnode->getAggregated(), aggrCalc.getVal(data)); + if constexpr (AggrCalcT::aggregate_over_values()) { + aggrCalc.add(lnode->getAggregated(), aggrCalc.getVal(data)); + } else { + aggrCalc.add(lnode->getAggregated(), aggrCalc.getVal(key)); + } ca = lnode->getAggregated(); } } diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp index 27a50e04f1b..98a14943d36 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp @@ -1089,12 +1089,12 @@ BTreeIterator:: updateData(const DataType & data, [[maybe_unused]] const AggrCalcT &aggrCalc) { LeafNodeType * lnode = getLeafNode(); - if constexpr (AggrCalcT::hasAggregated()) { + if constexpr (AggrCalcT::hasAggregated() && AggrCalcT::aggregate_over_values()) { AggrT oldca(lnode->getAggregated()); - typedef BTreeAggregator Aggregator; + using Aggregator = BTreeAggregator; if (aggrCalc.update(lnode->getAggregated(), aggrCalc.getVal(lnode->getData(_leaf.getIdx())), aggrCalc.getVal(data))) { @@ -1193,7 +1193,11 @@ insertFirst(const KeyType &key, const DataType &data, lnode.data->insert(0, key, data); if constexpr (AggrCalcT::hasAggregated()) { AggrT a; - aggrCalc.add(a, aggrCalc.getVal(data)); + if constexpr (AggrCalcT::aggregate_over_values()) { + aggrCalc.add(a, aggrCalc.getVal(data)); + } else { + aggrCalc.add(a, aggrCalc.getVal(key)); + } lnode.data->getAggregated() = a; } _leafRoot = lnode.data; diff --git a/vespalib/src/vespa/vespalib/btree/btreeremover.hpp b/vespalib/src/vespa/vespalib/btree/btreeremover.hpp index 6a9a7ebcc7e..468b24f2a81 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeremover.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeremover.hpp @@ -110,11 +110,17 @@ remove(BTreeNode::Ref &root, NodeAllocatorType &allocator(itr.getAllocator()); AggrT oldca(AggrCalcT::hasAggregated() ? lnode->getAggregated() : AggrT()); AggrT ca; - if (AggrCalcT::hasAggregated() && - aggrCalc.remove(lnode->getAggregated(), - aggrCalc.getVal(lnode->getData(idx)))) { + if constexpr (AggrCalcT::hasAggregated()) { + bool removed; + if constexpr (AggrCalcT::aggregate_over_values()) { + removed = aggrCalc.remove(lnode->getAggregated(), aggrCalc.getVal(lnode->getData(idx))); + } else { + removed = aggrCalc.remove(lnode->getAggregated(), aggrCalc.getVal(lnode->getKey(idx))); + } lnode->remove(idx); - Aggregator::recalc(*lnode, aggrCalc); + if (removed) { + Aggregator::recalc(*lnode, aggrCalc); + } } else { lnode->remove(idx); } diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.hpp b/vespalib/src/vespa/vespalib/btree/btreestore.hpp index ec3be588de4..e6c96600b70 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreestore.hpp @@ -959,7 +959,11 @@ getAggregated(const EntryRef ref) const const KeyDataType *shortArray = getKeyDataEntry(iRef, clusterSize); AggregatedType a; for (uint32_t i = 0; i < clusterSize; ++i) { - _aggrCalc.add(a, _aggrCalc.getVal(shortArray[i].getData())); + if constexpr (AggrCalcT::aggregate_over_values()) { + _aggrCalc.add(a, _aggrCalc.getVal(shortArray[i].getData())); + } else { + _aggrCalc.add(a, _aggrCalc.getVal(shortArray[i].getKey())); + } } return a; } diff --git a/vespalib/src/vespa/vespalib/btree/minmaxaggrcalc.h b/vespalib/src/vespa/vespalib/btree/minmaxaggrcalc.h index 66e23127c5a..6ef809861b7 100644 --- a/vespalib/src/vespa/vespalib/btree/minmaxaggrcalc.h +++ b/vespalib/src/vespa/vespalib/btree/minmaxaggrcalc.h @@ -11,6 +11,7 @@ class MinMaxAggrCalc public: constexpr MinMaxAggrCalc() = default; constexpr static bool hasAggregated() { return true; } + constexpr static bool aggregate_over_values() { return true; } static int32_t getVal(int32_t val) { return val; } static void add(MinMaxAggregated &a, int32_t val) { a.add(val); } static void add(MinMaxAggregated &a, const MinMaxAggregated &ca) { a.add(ca); } diff --git a/vespalib/src/vespa/vespalib/btree/noaggrcalc.h b/vespalib/src/vespa/vespalib/btree/noaggrcalc.h index 079abc57224..51958229430 100644 --- a/vespalib/src/vespa/vespalib/btree/noaggrcalc.h +++ b/vespalib/src/vespa/vespalib/btree/noaggrcalc.h @@ -17,6 +17,8 @@ public: return false; } + constexpr static bool aggregate_over_values() { return true; } + template static inline int32_t getVal(const DataT &val) -- cgit v1.2.3 From c58415566e23dcac5f0daa352f39f567a4d7b44f Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 23 Apr 2020 13:02:21 +0000 Subject: Use a single reloader per tls config file, and not one per instance. --- .../security/tls/ConfigFileBasedTlsContext.java | 158 +++------------------ .../java/com/yahoo/security/tls/TlsManager.java | 139 ++++++++++++++++++ 2 files changed, 162 insertions(+), 135 deletions(-) create mode 100644 security-utils/src/main/java/com/yahoo/security/tls/TlsManager.java diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java index 28854c59b2c..26dfbf9fd9f 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java @@ -1,32 +1,20 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.security.tls; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; -import com.yahoo.security.KeyUtils; import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.authz.PeerAuthorizerTrustManager; import com.yahoo.security.tls.policy.AuthorizedPeers; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLParameters; -import java.io.IOException; -import java.io.UncheckedIOException; import java.lang.ref.WeakReference; import java.nio.file.Path; -import java.security.KeyStore; -import java.time.Duration; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; /** * A {@link TlsContext} that uses the tls configuration specified in the transport security options file. @@ -36,12 +24,22 @@ import java.util.logging.Logger; */ public class ConfigFileBasedTlsContext implements TlsContext { - private static final Duration UPDATE_PERIOD = Duration.ofHours(1); + private final TlsContext tlsContext; + private TlsManager tlsManager; - private static final Logger log = Logger.getLogger(ConfigFileBasedTlsContext.class.getName()); + static private final Map> trustManagers = new HashMap<>(); - private final TlsContext tlsContext; - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ReloaderThreadFactory()); + private static TlsManager getOrCreateTrustManager(Path tlsOptionsConfigFile) { + synchronized (trustManagers) { + WeakReference tlsManager = trustManagers.get(tlsOptionsConfigFile); + if (tlsManager == null || tlsManager.get() == null) { + TlsManager manager = new TlsManager(tlsOptionsConfigFile); + trustManagers.put(tlsOptionsConfigFile, new WeakReference<>(manager)); + return manager; + } + return tlsManager.get(); + } + } public ConfigFileBasedTlsContext(Path tlsOptionsConfigFile, AuthorizationMode mode) { this(tlsOptionsConfigFile, mode, PeerAuthentication.NEED); @@ -52,56 +50,15 @@ public class ConfigFileBasedTlsContext implements TlsContext { * the TLS peer authentication is enforced at a higher protocol or application layer (e.g with {@link PeerAuthentication#WANT}). */ public ConfigFileBasedTlsContext(Path tlsOptionsConfigFile, AuthorizationMode mode, PeerAuthentication peerAuthentication) { - TransportSecurityOptions options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); - MutableX509TrustManager trustManager = new MutableX509TrustManager(); - MutableX509KeyManager keyManager = new MutableX509KeyManager(); - reloadTrustManager(options, trustManager); - reloadKeyManager(options, keyManager); - this.tlsContext = createDefaultTlsContext(options, mode, trustManager, keyManager, peerAuthentication); - this.scheduler.scheduleAtFixedRate(new CryptoMaterialReloader(tlsOptionsConfigFile, scheduler, trustManager, keyManager), - UPDATE_PERIOD.getSeconds()/*initial delay*/, - UPDATE_PERIOD.getSeconds(), - TimeUnit.SECONDS); - } - - private static void reloadTrustManager(TransportSecurityOptions options, MutableX509TrustManager trustManager) { - if (options.getCaCertificatesFile().isPresent()) { - trustManager.updateTruststore(loadTruststore(options.getCaCertificatesFile().get())); - } else { - trustManager.useDefaultTruststore(); - } - } - - private static void reloadKeyManager(TransportSecurityOptions options, MutableX509KeyManager keyManager) { - if (options.getPrivateKeyFile().isPresent() && options.getCertificatesFile().isPresent()) { - keyManager.updateKeystore(loadKeystore(options.getPrivateKeyFile().get(), options.getCertificatesFile().get()), new char[0]); - } else { - keyManager.useDefaultKeystore(); - } + tlsManager = getOrCreateTrustManager(tlsOptionsConfigFile); + this.tlsContext = createDefaultTlsContext(tlsManager.getOptions(), mode, tlsManager.getTrustManager(), tlsManager.getKeyManager(), peerAuthentication); } - private static KeyStore loadTruststore(Path caCertificateFile) { - try { - return KeyStoreBuilder.withType(KeyStoreType.PKCS12) - .withCertificateEntries("cert", X509CertificateUtils.certificateListFromPem(com.yahoo.vespa.jdk8compat.Files.readString(caCertificateFile))) - .build(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static KeyStore loadKeystore(Path privateKeyFile, Path certificatesFile) { - try { - return KeyStoreBuilder.withType(KeyStoreType.PKCS12) - .withKeyEntry( - "default", - KeyUtils.fromPemEncodedPrivateKey(com.yahoo.vespa.jdk8compat.Files.readString(privateKeyFile)), - X509CertificateUtils.certificateListFromPem(com.yahoo.vespa.jdk8compat.Files.readString(certificatesFile))) - .build(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } + // Wrapped methods from TlsContext + @Override public SSLContext context() { return tlsContext.context(); } + @Override public SSLParameters parameters() { return tlsContext.parameters(); } + @Override public SSLEngine createSslEngine() { return tlsContext.createSslEngine(); } + @Override public SSLEngine createSslEngine(String peerHost, int peerPort) { return tlsContext.createSslEngine(peerHost, peerPort); } private static DefaultTlsContext createDefaultTlsContext(TransportSecurityOptions options, AuthorizationMode mode, @@ -122,73 +79,4 @@ public class ConfigFileBasedTlsContext implements TlsContext { return new DefaultTlsContext(sslContext, ciphers, peerAuthentication); } - // Wrapped methods from TlsContext - @Override public SSLContext context() { return tlsContext.context(); } - @Override public SSLParameters parameters() { return tlsContext.parameters(); } - @Override public SSLEngine createSslEngine() { return tlsContext.createSslEngine(); } - @Override public SSLEngine createSslEngine(String peerHost, int peerPort) { return tlsContext.createSslEngine(peerHost, peerPort); } - - @Override - public void close() { - try { - scheduler.shutdownNow(); - if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) { - throw new RuntimeException("Unable to shutdown executor before timeout"); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - // Note: no reference to outer class (directly or indirectly) to ensure trust/key managers are eventually GCed once - // there are no more use of the outer class and the underlying SSLContext - private static class CryptoMaterialReloader implements Runnable { - - final Path tlsOptionsConfigFile; - final ScheduledExecutorService scheduler; - final WeakReference trustManager; - final WeakReference keyManager; - - CryptoMaterialReloader(Path tlsOptionsConfigFile, - ScheduledExecutorService scheduler, - MutableX509TrustManager trustManager, - MutableX509KeyManager keyManager) { - this.tlsOptionsConfigFile = tlsOptionsConfigFile; - this.scheduler = scheduler; - this.trustManager = new WeakReference<>(trustManager); - this.keyManager = new WeakReference<>(keyManager); - } - - @Override - public void run() { - try { - MutableX509TrustManager trustManager = this.trustManager.get(); - MutableX509KeyManager keyManager = this.keyManager.get(); - if (trustManager == null && keyManager == null) { - scheduler.shutdown(); - return; - } - TransportSecurityOptions options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); - if (trustManager != null) { - reloadTrustManager(options, trustManager); - } - if (keyManager != null) { - reloadKeyManager(options, keyManager); - } - } catch (Throwable t) { - log.log(Level.SEVERE, String.format("Failed to reload crypto material (path='%s'): %s", tlsOptionsConfigFile, t.getMessage()), t); - } - } - } - - // Static class to ensure no reference to outer class is contained - private static class ReloaderThreadFactory implements ThreadFactory { - @Override - public Thread newThread(Runnable r) { - Thread thread = new Thread(r, "tls-context-reloader"); - thread.setDaemon(true); - return thread; - } - } - } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TlsManager.java b/security-utils/src/main/java/com/yahoo/security/tls/TlsManager.java new file mode 100644 index 00000000000..bade1993982 --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/tls/TlsManager.java @@ -0,0 +1,139 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security.tls; + +import com.yahoo.security.KeyStoreBuilder; +import com.yahoo.security.KeyStoreType; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateUtils; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.ref.WeakReference; +import java.nio.file.Path; +import java.security.KeyStore; +import java.time.Duration; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +class TlsManager { + private static final Logger log = Logger.getLogger(TlsManager.class.getName()); + private static final Duration UPDATE_PERIOD = Duration.ofHours(1); + + private final MutableX509TrustManager trustManager; + private final MutableX509KeyManager keyManager; + private final ScheduledExecutorService scheduler; + private TransportSecurityOptions options; + + private static void reloadTrustManager(TransportSecurityOptions options, MutableX509TrustManager trustManager) { + if (options.getCaCertificatesFile().isPresent()) { + trustManager.updateTruststore(loadTruststore(options.getCaCertificatesFile().get())); + } else { + trustManager.useDefaultTruststore(); + } + } + + private static void reloadKeyManager(TransportSecurityOptions options, MutableX509KeyManager keyManager) { + if (options.getPrivateKeyFile().isPresent() && options.getCertificatesFile().isPresent()) { + keyManager.updateKeystore(loadKeystore(options.getPrivateKeyFile().get(), options.getCertificatesFile().get()), new char[0]); + } else { + keyManager.useDefaultKeystore(); + } + } + private static KeyStore loadTruststore(Path caCertificateFile) { + try { + return KeyStoreBuilder.withType(KeyStoreType.PKCS12) + .withCertificateEntries("cert", X509CertificateUtils.certificateListFromPem(com.yahoo.vespa.jdk8compat.Files.readString(caCertificateFile))) + .build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static KeyStore loadKeystore(Path privateKeyFile, Path certificatesFile) { + try { + return KeyStoreBuilder.withType(KeyStoreType.PKCS12) + .withKeyEntry( + "default", + KeyUtils.fromPemEncodedPrivateKey(com.yahoo.vespa.jdk8compat.Files.readString(privateKeyFile)), + X509CertificateUtils.certificateListFromPem(com.yahoo.vespa.jdk8compat.Files.readString(certificatesFile))) + .build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + // Note: no reference to outer class (directly or indirectly) to ensure trust/key managers are eventually GCed once + // there are no more use of the outer class and the underlying SSLContext + private static class CryptoMaterialReloader implements Runnable { + + private final Path tlsOptionsConfigFile; + private final ScheduledExecutorService scheduler; + private final WeakReference tlsManager; + + CryptoMaterialReloader(Path tlsOptionsConfigFile, + ScheduledExecutorService scheduler, + TlsManager tlsManager) { + this.tlsOptionsConfigFile = tlsOptionsConfigFile; + this.scheduler = scheduler; + this.tlsManager = new WeakReference<>(tlsManager); + } + + @Override + public void run() { + try { + TlsManager tlsManager = this.tlsManager.get(); + if (tlsManager == null) { + scheduler.shutdown(); + return; + } + TransportSecurityOptions options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); + reloadTrustManager(options, tlsManager.getTrustManager()); + MutableX509KeyManager keyManager = tlsManager.getKeyManager(); + reloadKeyManager(options, keyManager); + } catch (Throwable t) { + log.log(Level.SEVERE, String.format("Failed to reload crypto material (path='%s'): %s", tlsOptionsConfigFile, t.getMessage()), t); + } + } + } + + // Static class to ensure no reference to outer class is contained + private static class ReloaderThreadFactory implements ThreadFactory { + Path fileName; + ReloaderThreadFactory(Path fileName) { + this.fileName = fileName; + } + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r, "tls-context-reloader:" + fileName.toString() ); + thread.setDaemon(true); + return thread; + } + } + + TlsManager(Path tlsOptionsConfigFile) { + trustManager = new MutableX509TrustManager(); + keyManager = new MutableX509KeyManager(); + options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); + reloadTrustManager(options, trustManager); + reloadKeyManager(options, keyManager); + scheduler = Executors.newSingleThreadScheduledExecutor(new ReloaderThreadFactory(tlsOptionsConfigFile)); + this.scheduler.scheduleAtFixedRate(new CryptoMaterialReloader(tlsOptionsConfigFile, scheduler, this), + UPDATE_PERIOD.getSeconds()/*initial delay*/, + UPDATE_PERIOD.getSeconds(), + TimeUnit.SECONDS); + } + MutableX509TrustManager getTrustManager() { + return trustManager; + } + + MutableX509KeyManager getKeyManager() { + return keyManager; + } + + TransportSecurityOptions getOptions() { + return options; + } +} -- cgit v1.2.3 From 7fe35556f1e38558de96ddecb2f0f1c13dfff9a8 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Thu, 23 Apr 2020 15:51:53 +0200 Subject: Read bytes from word vector when writing dictionary files. --- .../vespa/searchlib/bitcompression/compression.cpp | 12 ++++++++++++ .../src/vespa/searchlib/bitcompression/compression.h | 10 +++++++++- .../src/vespa/searchlib/bitcompression/pagedict4.cpp | 19 +++---------------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index 83776d22fee..215ee6dabbe 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace search::bitcompression { @@ -263,6 +264,17 @@ writeBits(const uint64_t *bits, uint32_t bitOffset, uint32_t bitLength) } } +template +void +FeatureEncodeContext::writeBytes(vespalib::ConstArrayRef buf) +{ + for (unsigned char c : buf) { + writeBits(c, 8); + if (__builtin_expect(_valI >= _valE, false)) { + _writeContext->writeComprBuffer(false); + } + } +} template void diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h index 350932263c3..56dcee2751f 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h @@ -7,7 +7,12 @@ #include #include -namespace vespalib { class GenericHeader; } +namespace vespalib { + +class GenericHeader; +template class ConstArrayRef; + +} namespace search::index { class DocIdAndFeatures; } @@ -1651,6 +1656,9 @@ public: void writeBits(const uint64_t *bits, uint32_t bitOffset, uint32_t bitLength); + void + writeBytes(vespalib::ConstArrayRef buf); + void writeString(vespalib::stringref buf); diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp index 33f18d83af5..4f947c6c55d 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp @@ -5,6 +5,7 @@ #include "countcompression.h" #include #include +#include #include #include @@ -379,14 +380,7 @@ PageDict4SPWriter::flushPage() _prevL3Size - wordsSize * 8; e.padBits(padding); if (wordsSize > 0) { - // Pad with 7 NUL bytes to silence testing tools. - _words.reserve(_words.size() + 7); - memset(&*_words.end(), '\0', 7); - const char *wordsBufX = static_cast(&_words[0]); - size_t wordsBufXOff = reinterpret_cast(wordsBufX) & 7; - const uint64_t *wordsBuf = reinterpret_cast - (wordsBufX - wordsBufXOff); - e.writeBits(wordsBuf, 8 * wordsBufXOff, wordsSize * 8); + e.writeBytes(vespalib::ConstArrayRef(&_words[0], wordsSize)); } assert((e.getWriteOffset() & (getPageBitSize() - 1)) == 0); _l6Word = _l3Word; @@ -701,14 +695,7 @@ PageDict4PWriter::flushPage() _countsSize - _countsWordOffset * 8; e.padBits(padding); if (_countsWordOffset > 0) { - // Pad with 7 NUL bytes to silence testing tools. - _words.reserve(_words.size() + 7); - memset(&*_words.end(), '\0', 7); - const char *wordsBufX = static_cast(&_words[0]); - size_t wordsBufXOff = reinterpret_cast(wordsBufX) & 7; - const uint64_t *wordsBuf = reinterpret_cast - (wordsBufX - wordsBufXOff); - e.writeBits(wordsBuf, 8 * wordsBufXOff, _countsWordOffset * 8); + e.writeBytes(vespalib::ConstArrayRef(&_words[0], _countsWordOffset)); } assert((e.getWriteOffset() & (getPageBitSize() - 1)) == 0); _l3Word = _pendingCountsWord; -- cgit v1.2.3 From c3ee3ebc4e85e408a4be486e52745743bac85d0f Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Thu, 23 Apr 2020 14:13:37 +0000 Subject: Rename variable to be semantically accurate --- vespalib/src/vespa/vespalib/btree/btreeremover.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vespalib/src/vespa/vespalib/btree/btreeremover.hpp b/vespalib/src/vespa/vespalib/btree/btreeremover.hpp index 468b24f2a81..acb51b18b0c 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeremover.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeremover.hpp @@ -111,14 +111,14 @@ remove(BTreeNode::Ref &root, AggrT oldca(AggrCalcT::hasAggregated() ? lnode->getAggregated() : AggrT()); AggrT ca; if constexpr (AggrCalcT::hasAggregated()) { - bool removed; + bool need_aggregation_recalc; if constexpr (AggrCalcT::aggregate_over_values()) { - removed = aggrCalc.remove(lnode->getAggregated(), aggrCalc.getVal(lnode->getData(idx))); + need_aggregation_recalc = aggrCalc.remove(lnode->getAggregated(), aggrCalc.getVal(lnode->getData(idx))); } else { - removed = aggrCalc.remove(lnode->getAggregated(), aggrCalc.getVal(lnode->getKey(idx))); + need_aggregation_recalc = aggrCalc.remove(lnode->getAggregated(), aggrCalc.getVal(lnode->getKey(idx))); } lnode->remove(idx); - if (removed) { + if (need_aggregation_recalc) { Aggregator::recalc(*lnode, aggrCalc); } } else { -- cgit v1.2.3 From f8fba39ea9571981541126dd3b38f615b70d83d6 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Thu, 23 Apr 2020 16:44:52 +0200 Subject: Don't read beyond end of buffer returned by RawResultNode. --- searchlib/src/tests/aggregator/perdocexpr.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index 41465f991e3..039b3ae1fc0 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -762,7 +762,9 @@ TEST("testDivExpressions") { ToRawFunctionNode e(MU(MU(238686))); e.prepare(false); e.execute(); - EXPECT_EQUAL(strcmp(static_cast(e.getResult()).get().c_str(), "238686"), 0); + auto raw_result = static_cast(e.getResult()).get(); + EXPECT_EQUAL(6u, raw_result.size()); + EXPECT_EQUAL(strncmp(raw_result.c_str(), "238686", 6u), 0); } { -- cgit v1.2.3 From b9ceb17637499dee9370c29443b0f0d85b09638b Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 23 Apr 2020 15:50:43 +0000 Subject: To avoid deadlock when doing synchrounous execution in other chains, we use one threadpool per chain. --- docproc/abi-spec.json | 3 ++ .../docproc/AbstractConcreteDocumentFactory.java | 6 +-- .../java/com/yahoo/docproc/DocprocService.java | 38 ++++++++++++++++-- .../docproc/jdisc/DocumentProcessingHandler.java | 45 +++++++--------------- .../docproc/jdisc/DocumentProcessingTask.java | 14 ++++++- ...cumentProcessingTaskPrioritizationTestCase.java | 2 +- 6 files changed, 67 insertions(+), 41 deletions(-) diff --git a/docproc/abi-spec.json b/docproc/abi-spec.json index dee2d2172e4..116e2c9edfb 100644 --- a/docproc/abi-spec.json +++ b/docproc/abi-spec.json @@ -157,12 +157,15 @@ ], "methods": [ "public void (com.yahoo.component.ComponentId)", + "public void (com.yahoo.component.ComponentId, com.yahoo.docproc.CallStack, com.yahoo.document.DocumentTypeManager, int)", "public void (com.yahoo.component.ComponentId, com.yahoo.docproc.CallStack, com.yahoo.document.DocumentTypeManager)", "public void (java.lang.String)", + "public void deconstruct()", "public com.yahoo.document.DocumentTypeManager getDocumentTypeManager()", "public void setDocumentTypeManager(com.yahoo.document.DocumentTypeManager)", "public int getQueueSize()", "public com.yahoo.docproc.DocprocExecutor getExecutor()", + "public java.util.concurrent.ThreadPoolExecutor getThreadPoolExecutor()", "public void setInService(boolean)", "public boolean isInService()", "public boolean isAcceptingNewProcessings()", diff --git a/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java b/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java index 3e720f9e0aa..53ccbd4c325 100644 --- a/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java @@ -35,15 +35,15 @@ public abstract class AbstractConcreteDocumentFactory extends com.yahoo.componen /** * If the FieldValue is a StructuredFieldValue it will upgrade to the concrete type - * @param field - * @param fv + * @param field to upgrade + * @param fv value to upgrade * @return fv or upgraded fv */ public FieldValue optionallyUpgrade(Field field, FieldValue fv) { return optionallyUpgrade(field.getDataType(), fv); } - @SuppressWarnings({"unchecked", "rawtypes"}) + @SuppressWarnings({"unchecked"}) private FieldValue optionallyUpgrade(DataType dataType, FieldValue fv) { if (fv instanceof StructuredFieldValue) { try { diff --git a/docproc/src/main/java/com/yahoo/docproc/DocprocService.java b/docproc/src/main/java/com/yahoo/docproc/DocprocService.java index 766b433c4dd..b06e6bbdf40 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocprocService.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocprocService.java @@ -3,6 +3,7 @@ package com.yahoo.docproc; import com.yahoo.component.AbstractComponent; import com.yahoo.component.ComponentId; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.docproc.proxy.SchemaMap; import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentTypeManager; @@ -10,6 +11,8 @@ import com.yahoo.document.DocumentTypeManager; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -27,6 +30,7 @@ import java.util.logging.Logger; * * @author bratseth */ +//TODO Vespa 8 This class and a lot of other in this package should not be part of PublicAPI public class DocprocService extends AbstractComponent { private static Logger log = Logger.getLogger(DocprocService.class.getName()); @@ -34,6 +38,7 @@ public class DocprocService extends AbstractComponent { /** The processings currently in progress at this service */ private final LinkedBlockingQueue queue; + private final ThreadPoolExecutor threadPool; /** The current state of this service */ private boolean inService = false; /** The current state of this service */ @@ -41,9 +46,18 @@ public class DocprocService extends AbstractComponent { public static SchemaMap schemaMap = new SchemaMap(); private DocumentTypeManager documentTypeManager = null; - public DocprocService(ComponentId id) { + private DocprocService(ComponentId id, int numThreads) { super(id); queue = new LinkedBlockingQueue<>(); + threadPool = new ThreadPoolExecutor(numThreads, + numThreads, + 0, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + new DaemonThreadFactory("docproc-" + id.stringValue() + "-")); + } + + public DocprocService(ComponentId id) { + this(id, Runtime.getRuntime().availableProcessors()); } /** @@ -52,20 +66,32 @@ public class DocprocService extends AbstractComponent { * @param id the component id of the new service. * @param stack the call stack to use. * @param mgr the document type manager to use. + * @param numThreads to have in the thread pool */ - public DocprocService(ComponentId id, CallStack stack, DocumentTypeManager mgr) { - this(id); + public DocprocService(ComponentId id, CallStack stack, DocumentTypeManager mgr, int numThreads) { + this(id, numThreads); setCallStack(stack); setDocumentTypeManager(mgr); setInService(true); } + @Deprecated + public DocprocService(ComponentId id, CallStack stack, DocumentTypeManager mgr) { + this(id, stack, mgr, Runtime.getRuntime().availableProcessors()); + } + /** * Creates a service with a name with an unbounded input queue. If the given name is null or the empty string, * it will become the name "default". + * Testing only */ public DocprocService(String name) { - this(new ComponentId(name, null)); + this(new ComponentId(name, null), 1); + } + + @Override + public void deconstruct() { + threadPool.shutdown(); } public DocumentTypeManager getDocumentTypeManager() { @@ -90,6 +116,10 @@ public class DocprocService extends AbstractComponent { return executor; } + public ThreadPoolExecutor getThreadPoolExecutor() { + return threadPool; + } + private void setExecutor(DocprocExecutor executor) { this.executor = executor; } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index d34c300b8d0..f93598058cf 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -30,12 +30,7 @@ import com.yahoo.processing.execution.chain.ChainRegistry; import com.yahoo.statistics.Statistics; import java.util.TimerTask; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -53,7 +48,6 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { private final ComponentRegistry docprocServiceRegistry; private final ComponentRegistry docFactoryRegistry; private final ChainRegistry chainRegistry = new ChainRegistry<>(); - private ThreadPoolExecutor threadPool; private final ScheduledThreadPoolExecutor laterExecutor = new ScheduledThreadPoolExecutor(2, new DaemonThreadFactory("docproc-later-")); private ContainerDocumentConfig containerDocConfig; @@ -62,17 +56,16 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { public DocumentProcessingHandler(ComponentRegistry docprocServiceRegistry, ComponentRegistry documentProcessorComponentRegistry, ComponentRegistry docFactoryRegistry, - ThreadPoolExecutor threadPool, DocumentTypeManager documentTypeManager, + int numThreads, + DocumentTypeManager documentTypeManager, ChainsModel chainsModel, SchemaMap schemaMap, Statistics statistics, Metric metric, ContainerDocumentConfig containerDocConfig) { this.docprocServiceRegistry = docprocServiceRegistry; this.docFactoryRegistry = docFactoryRegistry; - this.threadPool = threadPool; this.containerDocConfig = containerDocConfig; this.documentTypeManager = documentTypeManager; DocprocService.schemaMap = schemaMap; - threadPool.prestartCoreThread(); laterExecutor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); laterExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); @@ -81,12 +74,12 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { for (Chain chain : chainRegistry.allComponents()) { log.config("Setting up call stack for chain " + chain.getId()); - DocprocService service = - new DocprocService(chain.getId(), convertToCallStack(chain, statistics, metric), documentTypeManager); + DocprocService service = new DocprocService(chain.getId(), convertToCallStack(chain, statistics, metric), documentTypeManager, computeNumThreads(numThreads)); service.setInService(true); docprocServiceRegistry.register(service.getId(), service); } } + } private static int computeNumThreads(int maxThreads) { @@ -98,12 +91,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { ComponentRegistry docFactoryRegistry, DocumentProcessingHandlerParameters params) { this(docprocServiceRegistry, documentProcessorComponentRegistry, docFactoryRegistry, - new ThreadPoolExecutor(computeNumThreads(params.getMaxNumThreads()), - computeNumThreads(params.getMaxNumThreads()), - 0,TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - new DaemonThreadFactory("docproc-") - ), + params.getMaxNumThreads(), params.getDocumentTypeManager(), params.getChainsModel(), params.getSchemaMap(), params.getStatisticsManager(), params.getMetric(), @@ -133,7 +121,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { @Override protected void destroy() { - threadPool.shutdown(); //calling shutdownNow() seems like a bit of an overkill + //threadPoolMap.values().forEach( pool -> pool.shutdown()); //calling shutdownNow() seems like a bit of an overkill } public ComponentRegistry getDocprocServiceRegistry() { @@ -175,13 +163,14 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { return null; } - DocprocService service = docprocServiceRegistry.getComponent(requestContext.getServiceName()); + String serviceName = requestContext.getServiceName(); + DocprocService service = docprocServiceRegistry.getComponent(serviceName); //No need to enqueue a task if the docproc chain is empty, just forward requestContext if (service == null) { - log.log(LogLevel.ERROR, "DocprocService for session '" + requestContext.getServiceName() + + log.log(LogLevel.ERROR, "DocprocService for session '" + serviceName + "' not found, returning request '" + requestContext + "'."); requestContext.processingFailed(RequestContext.ErrorCode.ERROR_PROCESSING_FAILURE, - "DocprocService " + requestContext.getServiceName() + " not found."); + "DocprocService " + serviceName + " not found."); return null; } else if (service.getExecutor().getCallStack().size() == 0) { //call stack was empty, just forward message @@ -189,19 +178,11 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { return null; } - DocumentProcessingTask task = new DocumentProcessingTask(requestContext, this, service); - submit(task); + DocumentProcessingTask task = new DocumentProcessingTask(requestContext, this, service, service.getThreadPoolExecutor()); + task.submit(); return null; } - private void submit(DocumentProcessingTask task) { - try { - threadPool.execute(task); - } catch (RejectedExecutionException ree) { - task.queueFull(); - } - } - void submit(DocumentProcessingTask task, long delay) { LaterTimerTask timerTask = new LaterTimerTask(task, delay); laterExecutor.schedule(timerTask, delay, TimeUnit.MILLISECONDS); @@ -220,7 +201,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { @Override public void run() { log.log(LogLevel.DEBUG, "Submitting after having waited " + delay + " ms in LATER queue: " + processingTask); - submit(processingTask); + processingTask.submit(); } } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index ca4648678a5..4d887616093 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -17,6 +17,8 @@ import java.io.StringWriter; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -36,13 +38,23 @@ public class DocumentProcessingTask implements Comparable Date: Thu, 23 Apr 2020 18:41:55 +0200 Subject: Fix taskinfo printing in storageframework ticking thread test. --- storageframework/src/tests/thread/tickingthreadtest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storageframework/src/tests/thread/tickingthreadtest.cpp b/storageframework/src/tests/thread/tickingthreadtest.cpp index f69f77e8a07..8e3d7b29cd6 100644 --- a/storageframework/src/tests/thread/tickingthreadtest.cpp +++ b/storageframework/src/tests/thread/tickingthreadtest.cpp @@ -273,7 +273,7 @@ struct BroadcastApp : public TickingThread { ThreadWaitInfo doNonCriticalTick(ThreadIndex) override { if (!_active.empty()) { for (uint32_t i=0; i<_active.size(); ++i) { - printTaskInfo(_queue[i], "processing"); + printTaskInfo(_active[i], "processing"); _processed.push_back(_active[i]); } _active.clear(); -- cgit v1.2.3 From 1c6c89eb52ac80c583c0cd90efdd0784344af434 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 23 Apr 2020 16:47:57 +0000 Subject: Use reference counting to avoid relying on GC to drop threads. --- .../security/tls/ConfigFileBasedTlsContext.java | 25 ++++++++++++++++------ .../java/com/yahoo/security/tls/TlsManager.java | 19 ++++++++++++++-- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java index 26dfbf9fd9f..9527d50f339 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java @@ -31,13 +31,17 @@ public class ConfigFileBasedTlsContext implements TlsContext { private static TlsManager getOrCreateTrustManager(Path tlsOptionsConfigFile) { synchronized (trustManagers) { - WeakReference tlsManager = trustManagers.get(tlsOptionsConfigFile); - if (tlsManager == null || tlsManager.get() == null) { - TlsManager manager = new TlsManager(tlsOptionsConfigFile); - trustManagers.put(tlsOptionsConfigFile, new WeakReference<>(manager)); - return manager; + WeakReference tlsRef = trustManagers.get(tlsOptionsConfigFile); + TlsManager tlsManager = null; + if (tlsRef != null) { + tlsManager = tlsRef.get(); } - return tlsManager.get(); + if (tlsManager == null) { + tlsManager = new TlsManager(tlsOptionsConfigFile); + trustManagers.put(tlsOptionsConfigFile, new WeakReference<>(tlsManager)); + } + tlsManager.addRef(); + return tlsManager; } } @@ -59,6 +63,15 @@ public class ConfigFileBasedTlsContext implements TlsContext { @Override public SSLParameters parameters() { return tlsContext.parameters(); } @Override public SSLEngine createSslEngine() { return tlsContext.createSslEngine(); } @Override public SSLEngine createSslEngine(String peerHost, int peerPort) { return tlsContext.createSslEngine(peerHost, peerPort); } + @Override public void close() { + synchronized (trustManagers) { + int references = tlsManager.subRef(); + if (references == 0) { + tlsManager.close(); + trustManagers.remove(tlsManager.getTlsConfigFile()); + } + } + } private static DefaultTlsContext createDefaultTlsContext(TransportSecurityOptions options, AuthorizationMode mode, diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TlsManager.java b/security-utils/src/main/java/com/yahoo/security/tls/TlsManager.java index bade1993982..c2286c3e8ac 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TlsManager.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TlsManager.java @@ -16,6 +16,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; @@ -23,10 +24,12 @@ class TlsManager { private static final Logger log = Logger.getLogger(TlsManager.class.getName()); private static final Duration UPDATE_PERIOD = Duration.ofHours(1); + private final Path tlsConfigFile; private final MutableX509TrustManager trustManager; private final MutableX509KeyManager keyManager; private final ScheduledExecutorService scheduler; private TransportSecurityOptions options; + private final AtomicInteger references = new AtomicInteger(0); private static void reloadTrustManager(TransportSecurityOptions options, MutableX509TrustManager trustManager) { if (options.getCaCertificatesFile().isPresent()) { @@ -86,13 +89,13 @@ class TlsManager { try { TlsManager tlsManager = this.tlsManager.get(); if (tlsManager == null) { + // If reference count is done correctly this should not be necessary. scheduler.shutdown(); return; } TransportSecurityOptions options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); reloadTrustManager(options, tlsManager.getTrustManager()); - MutableX509KeyManager keyManager = tlsManager.getKeyManager(); - reloadKeyManager(options, keyManager); + reloadKeyManager(options, tlsManager.getKeyManager()); } catch (Throwable t) { log.log(Level.SEVERE, String.format("Failed to reload crypto material (path='%s'): %s", tlsOptionsConfigFile, t.getMessage()), t); } @@ -114,6 +117,7 @@ class TlsManager { } TlsManager(Path tlsOptionsConfigFile) { + tlsConfigFile = tlsOptionsConfigFile; trustManager = new MutableX509TrustManager(); keyManager = new MutableX509KeyManager(); options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); @@ -133,7 +137,18 @@ class TlsManager { return keyManager; } + Path getTlsConfigFile() { + return tlsConfigFile; + } + TransportSecurityOptions getOptions() { return options; } + + void close() { + scheduler.shutdown(); + } + + int addRef() { return references.incrementAndGet(); } + int subRef() { return references.decrementAndGet(); } } -- cgit v1.2.3 From af7cb053e10811093700326e8d3768f61158fd4c Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Thu, 23 Apr 2020 18:48:31 +0200 Subject: Don't write outside container in vdslib distribution test. --- vdslib/src/tests/distribution/distributiontest.cpp | 37 +++++++++++----------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index c43735e7e41..f0b48faebef 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -932,9 +932,9 @@ TEST(DistributionTest, test_hierarchical_no_redistribution) std::vector nodes; std::vector< std::vector > distr(4); - int numBuckets = 1000; + size_t numBuckets = 1000; - for (int i = 0; i < numBuckets; i++) { + for (size_t i = 0; i < numBuckets; i++) { nodes = distribution.getIdealStorageNodes( state, document::BucketId(16, i), "u"); for (uint16_t j=0; j::iterator it; - std::vector v(1000); + std::vector v; - it=set_intersection (distr[0].begin(), distr[0].end(), distr[1].begin(), distr[1].end(), v.begin()); - EXPECT_EQ(0, int(it-v.begin())); + set_intersection (distr[0].begin(), distr[0].end(), distr[1].begin(), distr[1].end(), back_inserter(v)); + EXPECT_EQ(0, v.size()); v.clear(); - it=set_intersection (distr[2].begin(), distr[2].end(), distr[3].begin(), distr[3].end(), v.begin()); - EXPECT_EQ(0, int(it-v.begin())); + set_intersection (distr[2].begin(), distr[2].end(), distr[3].begin(), distr[3].end(), back_inserter(v)); + EXPECT_EQ(0, v.size()); v.clear(); - it=set_union (distr[0].begin(), distr[0].end(), distr[1].begin(), distr[1].end(), v.begin()); - EXPECT_EQ(numBuckets, int(it-v.begin())); + set_union (distr[0].begin(), distr[0].end(), distr[1].begin(), distr[1].end(), back_inserter(v)); + EXPECT_EQ(numBuckets, v.size()); v.clear(); - it=set_union (distr[2].begin(), distr[2].end(), distr[3].begin(), distr[3].end(), v.begin()); - EXPECT_EQ(numBuckets, int(it-v.begin())); + set_union (distr[2].begin(), distr[2].end(), distr[3].begin(), distr[3].end(), back_inserter(v)); + EXPECT_EQ(numBuckets, v.size()); v.clear(); state.setNodeState(Node(NodeType::STORAGE, 0), @@ -967,7 +966,7 @@ TEST(DistributionTest, test_hierarchical_no_redistribution) std::vector< std::vector > distr2(4); - for (int i = 0; i < numBuckets; i++) { + for (size_t i = 0; i < numBuckets; i++) { nodes = distribution.getIdealStorageNodes( state, document::BucketId(16, i), "u"); for (uint16_t j=0; j Date: Thu, 23 Apr 2020 19:16:25 +0200 Subject: If compiler is clang then disable the clock computation hack. --- vespalib/src/vespa/vespalib/util/time.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vespalib/src/vespa/vespalib/util/time.cpp b/vespalib/src/vespa/vespalib/util/time.cpp index 61a295c6dc3..341102bf9ff 100644 --- a/vespalib/src/vespa/vespalib/util/time.cpp +++ b/vespalib/src/vespa/vespalib/util/time.cpp @@ -53,6 +53,8 @@ Timer::waitAtLeast(duration dur, bool busyWait) { } +#ifndef __clang__ + namespace std::chrono { /* @@ -83,3 +85,4 @@ steady_clock::now() noexcept { } } +#endif -- cgit v1.2.3 From efb707280771ee43a53320ee15cd49f341370605 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Thu, 23 Apr 2020 19:34:57 +0200 Subject: Use java.util.logging.Level instead of vespalog's LogLevel. - The purpose is to allow using config in vespalog. --- config/pom.xml | 2 +- .../subscription/CfgConfigPayloadBuilder.java | 8 +++-- .../com/yahoo/config/subscription/ConfigDebug.java | 5 ++-- .../yahoo/config/subscription/ConfigSourceSet.java | 5 ++-- .../config/subscription/ConfigSubscriber.java | 11 ++++--- .../subscription/impl/FileConfigSubscription.java | 17 ++++++----- .../impl/GenericJRTConfigSubscription.java | 5 ++-- .../subscription/impl/JRTConfigRequester.java | 35 ++++++++++++---------- .../subscription/impl/JRTConfigSubscription.java | 21 ++++++------- .../yahoo/vespa/config/ConfigPayloadApplier.java | 15 ++++++---- .../com/yahoo/vespa/config/JRTConnectionPool.java | 5 ++-- .../java/com/yahoo/vespa/config/UrlDownloader.java | 10 ++++--- .../vespa/config/protocol/RequestValidation.java | 17 ++++++----- 13 files changed, 88 insertions(+), 68 deletions(-) diff --git a/config/pom.xml b/config/pom.xml index ad036c442d1..6e4e26ed0f1 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -53,7 +53,7 @@ com.yahoo.vespa - vespalog + defaults ${project.version} diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java index 474c9f7a4db..74d4d378e5d 100644 --- a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java +++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java @@ -4,13 +4,15 @@ package com.yahoo.config.subscription; import com.yahoo.collections.Pair; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.StringNode; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; import java.util.ArrayList; import java.util.List; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.FINEST; + /** * Deserializes config payload (cfg format) to a ConfigPayload. * @@ -37,7 +39,7 @@ public class CfgConfigPayloadBuilder { parseLine(line, lineNum, payloadBuilder); lineNum++; } - log.log(LogLevel.SPAM, () -> "payload=" + payloadBuilder.toString()); + log.log(FINEST, () -> "payload=" + payloadBuilder.toString()); return payloadBuilder; } @@ -48,7 +50,7 @@ public class CfgConfigPayloadBuilder { String field = fieldAndValue.getFirst(); String value = fieldAndValue.getSecond(); if (field==null || value==null) { - log.log(LogLevel.DEBUG, () -> "Got field without value in line " + lineNum + ": " + line + ", skipping"); + log.log(FINE, () -> "Got field without value in line " + lineNum + ": " + line + ", skipping"); return; } field=field.trim(); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java index 3020f49b32e..d279fc42251 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java @@ -2,16 +2,17 @@ package com.yahoo.config.subscription; import com.yahoo.config.ConfigInstance; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import java.util.logging.Logger; +import static java.util.logging.Level.INFO; + // Debug class that provides useful helper routines public class ConfigDebug { public static void logDebug(Logger logger, long timestamp, ConfigKey key, String logmessage) { if (key.getConfigId().matches(".*container.?\\d+.*") || key.getConfigId().matches(".*doc.api.*")) { - logger.log(LogLevel.INFO, timestamp + " " + key + " " + logmessage); + logger.log(INFO, timestamp + " " + key + " " + logmessage); } } diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java index 7472439d6a4..e0605dce30d 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import com.yahoo.log.LogLevel; - import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; @@ -10,6 +8,7 @@ import java.util.List; import java.util.Set; import java.util.logging.Logger; +import static java.util.logging.Level.INFO; /** * An immutable set of connection endpoints, where each endpoint points to either a @@ -98,7 +97,7 @@ public class ConfigSourceSet implements ConfigSource { public static ConfigSourceSet createDefault() { String configSources = System.getenv("VESPA_CONFIG_SOURCES"); if (configSources != null) { - log.log(LogLevel.INFO, "Using config sources from VESPA_CONFIG_SOURCES: " + configSources); + log.log(INFO, "Using config sources from VESPA_CONFIG_SOURCES: " + configSources); return new ConfigSourceSet(checkSourcesSyntax(configSources)); } else { String[] def = {"tcp/localhost:" + System.getProperty("vespa.config.port", "19090")}; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index 3891d710fa3..814222989a1 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -5,7 +5,6 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.impl.ConfigSubscription; import com.yahoo.config.subscription.impl.JRTConfigRequester; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.TimingValues; import com.yahoo.yolean.Exceptions; @@ -14,8 +13,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; + import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.SEVERE; +import static java.util.logging.Level.WARNING; import static java.util.stream.Collectors.toList; /** @@ -326,7 +329,7 @@ public class ConfigSubscriber implements AutoCloseable { h.subscription().close(); } closeRequesters(); - log.log(LogLevel.DEBUG, () -> "Config subscriber has been closed."); + log.log(FINE, () -> "Config subscriber has been closed."); } /** @@ -434,7 +437,7 @@ public class ConfigSubscriber implements AutoCloseable { if (handle.isChanged()) singleSubscriber.configure(handle.getConfig()); } } catch (Exception e) { - log.log(LogLevel.ERROR, "Exception from config system, continuing config thread: " + Exceptions.toMessageString(e)); + log.log(SEVERE, "Exception from config system, continuing config thread: " + Exceptions.toMessageString(e)); } } } @@ -477,7 +480,7 @@ public class ConfigSubscriber implements AutoCloseable { protected void finalize() throws Throwable { try { if (!isClosed()) { - log.log(LogLevel.WARNING, stackTraceAtConstruction, + log.log(WARNING, stackTraceAtConstruction, () -> String.format("%s: Closing subscription from finalizer() - close() has not been called (keys=%s)", super.toString(), subscriptionHandles.stream().map(handle -> handle.subscription().getKey().toString()).collect(toList()))); diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java index 60c920d1dfa..365f6a4d4c5 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java @@ -1,10 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; -import java.io.File; -import java.io.IOException; -import java.util.Arrays; - import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.CfgConfigPayloadBuilder; @@ -13,7 +9,12 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.io.IOUtils; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.log.LogLevel; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import static java.util.logging.Level.FINE; /** * Subscription used when config id is file:... @@ -37,12 +38,12 @@ public class FileConfigSubscription extends ConfigSubs public boolean nextConfig(long timeout) { if (!file.exists() && !file.isFile()) throw new IllegalArgumentException("Not a file: "+file); if (checkReloaded()) { - log.log(LogLevel.DEBUG, "User forced config reload at " + System.currentTimeMillis()); + log.log(FINE, "User forced config reload at " + System.currentTimeMillis()); // User forced reload setConfigIfChanged(updateConfig()); ConfigState configState = getConfigState(); - log.log(LogLevel.DEBUG, "Config updated at " + System.currentTimeMillis() + ", changed: " + configState.isConfigChanged()); - log.log(LogLevel.DEBUG, "Config: " + configState.getConfig().toString()); + log.log(FINE, "Config updated at " + System.currentTimeMillis() + ", changed: " + configState.isConfigChanged()); + log.log(FINE, "Config: " + configState.getConfig().toString()); return true; } if (file.lastModified()!=ts) { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java index 8ce3449fba5..eec18b93e71 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -5,13 +5,14 @@ import java.util.List; import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; +import static java.util.logging.Level.FINE; + /** * A JRT subscription which does not use the config class, but {@link com.yahoo.vespa.config.RawConfig} instead. * Used by config proxy. @@ -33,7 +34,7 @@ public class GenericJRTConfigSubscription extends JRTConfigSubscription "in setNewConfig, config=" + this.getConfigState().getConfig()); + log.log(FINE, () -> "in setNewConfig, config=" + this.getConfigState().getConfig()); } // This method is overridden because config needs to have its generation diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 49c5dcd343c..6a43c397562 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -6,7 +6,6 @@ import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Request; import com.yahoo.jrt.RequestWaiter; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.ErrorCode; @@ -24,6 +23,12 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.FINEST; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.SEVERE; +import static java.util.logging.Level.WARNING; + /** * This class fetches config payload using JRT, and acts as the callback target. * It uses the {@link JRTConfigSubscription} and {@link JRTClientConfigRequest} @@ -90,9 +95,9 @@ public class JRTConfigRequester implements RequestWaiter { if ( ! req.validateParameters()) throw new ConfigurationRuntimeException("Error in parameters for config request: " + req); double jrtClientTimeout = getClientTimeout(req); - log.log(LogLevel.DEBUG, () -> "Requesting config for " + sub + " on connection " + connection + log.log(FINE, () -> "Requesting config for " + sub + " on connection " + connection + " with client timeout " + jrtClientTimeout + - (log.isLoggable(LogLevel.SPAM) ? (",defcontent=" + req.getDefContent().asString()) : "")); + (log.isLoggable(FINEST) ? (",defcontent=" + req.getDefContent().asString()) : "")); connection.invokeAsync(req.getRequest(), jrtClientTimeout, this); } @@ -110,7 +115,7 @@ public class JRTConfigRequester implements RequestWaiter { sub.setException(e); } else { // Very unlikely - log.log(Level.SEVERE, "Failed to get subscription object from JRT config callback: " + + log.log(SEVERE, "Failed to get subscription object from JRT config callback: " + Exceptions.toMessageString(e)); } } @@ -118,11 +123,11 @@ public class JRTConfigRequester implements RequestWaiter { private void doHandle(JRTConfigSubscription sub, JRTClientConfigRequest jrtReq, Connection connection) { boolean validResponse = jrtReq.validateResponse(); - log.log(LogLevel.DEBUG, () -> "Request callback " + (validResponse ? "valid" : "invalid") + ". Req: " + jrtReq + "\nSpec: " + connection); + log.log(FINE, () -> "Request callback " + (validResponse ? "valid" : "invalid") + ". Req: " + jrtReq + "\nSpec: " + connection); if (sub.getState() == ConfigSubscription.State.CLOSED) return; // Avoid error messages etc. after closing Trace trace = jrtReq.getResponseTrace(); trace.trace(TRACELEVEL, "JRTConfigRequester.doHandle()"); - log.log(LogLevel.SPAM, () -> trace.toString()); + log.log(FINEST, () -> trace.toString()); if (validResponse) { handleOKRequest(jrtReq, sub, connection); } else { @@ -134,20 +139,20 @@ public class JRTConfigRequester implements RequestWaiter { private void logWhenErrorResponse(JRTClientConfigRequest jrtReq, Connection connection) { switch (jrtReq.errorCode()) { case com.yahoo.jrt.ErrorCode.CONNECTION: - log.log(LogLevel.DEBUG, () -> "Request callback failed: " + jrtReq.errorMessage() + + log.log(FINE, () -> "Request callback failed: " + jrtReq.errorMessage() + "\nConnection spec: " + connection); break; case ErrorCode.APPLICATION_NOT_LOADED: case ErrorCode.UNKNOWN_VESPA_VERSION: if (noApplicationWarningLogged.isBefore(Instant.now().minus(delayBetweenWarnings))) { - log.log(LogLevel.WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + + log.log(WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + ". Connection spec: " + connection.getAddress() + ", error message: " + jrtReq.errorMessage()); noApplicationWarningLogged = Instant.now(); } break; default: - log.log(LogLevel.WARNING, "Request callback failed. Req: " + jrtReq + "\nSpec: " + connection.getAddress() + + log.log(WARNING, "Request callback failed. Req: " + jrtReq + "\nSpec: " + connection.getAddress() + " . Req error message: " + jrtReq.errorMessage()); break; } @@ -157,7 +162,7 @@ public class JRTConfigRequester implements RequestWaiter { final boolean configured = (sub.getConfigState().getConfig() != null); if (configured) { // The subscription object has an "old" config, which is all we have to offer back now - log.log(LogLevel.INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); + log.log(INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); } ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode()); connectionPool.setError(connection, jrtReq.errorCode()); @@ -192,7 +197,7 @@ public class JRTConfigRequester implements RequestWaiter { Connection connection) { transientFailures++; if (suspendWarningLogged.isBefore(Instant.now().minus(delayBetweenWarnings))) { - log.log(LogLevel.INFO, "Connection to " + connection.getAddress() + + log.log(INFO, "Connection to " + connection.getAddress() + " failed or timed out, clients will keep existing config, will keep trying."); suspendWarningLogged = Instant.now(); } @@ -218,7 +223,7 @@ public class JRTConfigRequester implements RequestWaiter { if (sub.getState() != ConfigSubscription.State.OPEN) return; fatalFailures++; // The logging depends on whether we are configured or not. - Level logLevel = sub.getConfigState().getConfig() == null ? LogLevel.DEBUG : LogLevel.INFO; + Level logLevel = sub.getConfigState().getConfig() == null ? FINE : INFO; String logMessage = "Request for config " + jrtReq.getShortDescription() + "' failed with error code " + jrtReq.errorCode() + " (" + jrtReq.errorMessage() + "), scheduling new connect " + " in " + delay + " ms"; @@ -236,7 +241,7 @@ public class JRTConfigRequester implements RequestWaiter { noApplicationWarningLogged = Instant.MIN; connection.setSuccess(); sub.setLastCallBackOKTS(Instant.now()); - log.log(LogLevel.DEBUG, () -> "OK response received in handleOkRequest: " + jrtReq); + log.log(FINE, () -> "OK response received in handleOkRequest: " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { // We only want this latest generation to be in the queue, we do not preserve history in this system sub.getReqQueue().clear(); @@ -260,8 +265,8 @@ public class JRTConfigRequester implements RequestWaiter { private void scheduleNextRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription sub, long delay, long timeout) { long delayBeforeSendingRequest = (delay < 0) ? 0 : delay; JRTClientConfigRequest jrtReqNew = jrtReq.nextRequest(timeout); - log.log(LogLevel.SPAM, () -> timingValues.toString()); - log.log(LogLevel.DEBUG, () -> "Scheduling new request " + delayBeforeSendingRequest + " millis from now for " + jrtReqNew.getConfigKey()); + log.log(FINEST, () -> timingValues.toString()); + log.log(FINE, () -> "Scheduling new request " + delayBeforeSendingRequest + " millis from now for " + jrtReqNew.getConfigKey()); scheduler.schedule(new GetConfigTask(jrtReqNew, sub), delayBeforeSendingRequest, TimeUnit.MILLISECONDS); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index a94a135f9d8..44f6d65ee65 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -1,25 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.ConfigInterruptedException; import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.JRTConnectionPool; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.CompressionType; import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; import com.yahoo.vespa.config.protocol.Payload; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.INFO; + /** * A JRT config subscription uses one {@link JRTConfigRequester} to fetch config using Vespa RPC from a config source, typically proxy or server * @@ -90,7 +91,7 @@ public class JRTConfigSubscription extends ConfigSubsc // timed out, we know nothing new. return false; } - log.log(LogLevel.DEBUG, () -> "Polled queue and found config " + jrtReq); + log.log(FINE, () -> "Polled queue and found config " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { setInternalRedeploy(jrtReq.responseIsInternalRedeploy()); if (jrtReq.hasUpdatedConfig()) { @@ -192,7 +193,7 @@ public class JRTConfigSubscription extends ConfigSubsc @Override public void reload(long generation) { - log.log(LogLevel.DEBUG, "reload() is without effect on a JRTConfigSubscription."); + log.log(FINE, "reload() is without effect on a JRTConfigSubscription."); } void setLastCallBackOKTS(Instant lastCallBackOKTS) { @@ -204,7 +205,7 @@ public class JRTConfigSubscription extends ConfigSubsc static void printStatus(JRTClientConfigRequest request, String message) { final String name = request.getConfigKey().getName(); if (name.equals("components") || name.equals("chains")) { - log.log(LogLevel.INFO, message + ":" + name + ":" + ", request=" + request); + log.log(INFO, message + ":" + name + ":" + ", request=" + request); } } } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 134352736b6..b4df42c802e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -5,13 +5,12 @@ import com.yahoo.config.ConfigBuilder; import com.yahoo.config.ConfigInstance; import com.yahoo.config.FileReference; import com.yahoo.config.UrlReference; -import com.yahoo.log.LogLevel; -import com.yahoo.yolean.Exceptions; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Type; import com.yahoo.text.Utf8; +import com.yahoo.yolean.Exceptions; import java.io.File; import java.lang.reflect.Constructor; @@ -27,6 +26,10 @@ import java.util.Set; import java.util.Stack; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.FINEST; +import static java.util.logging.Level.INFO; + /** * A utility class that can be used to apply a payload to a config builder. * @@ -224,7 +227,7 @@ public class ConfigPayloadApplier { } catch (InvocationTargetException | IllegalAccessException e) { throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); } catch (NoSuchMethodException e) { - log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + rootBuilder); + log.log(INFO, "Skipping unknown field " + methodName + " in " + rootBuilder); } } @@ -279,7 +282,7 @@ public class ConfigPayloadApplier { } catch (InvocationTargetException | IllegalAccessException e) { throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); } catch (NoSuchMethodException e) { - log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + builder.getClass()); + log.log(INFO, "Skipping unknown field " + methodName + " in " + builder.getClass()); } } @@ -479,11 +482,11 @@ public class ConfigPayloadApplier { } private void debug(String message) { - log.log(LogLevel.DEBUG, () -> message); + log.log(FINE, () -> message); } private void trace(String message) { - log.log(LogLevel.SPAM, () -> message); + log.log(FINEST, () -> message); } private void printStack() { diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java index 326c1287468..2c1d00f295e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.config; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -13,6 +12,8 @@ import java.util.Map; import java.util.concurrent.ThreadLocalRandom; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; + /** * A pool of JRT connections to a config source (either a config server or a config proxy). * The current connection is chosen randomly when calling {#link #setNewCurrentConnection} @@ -74,7 +75,7 @@ public class JRTConnectionPool implements ConnectionPool { public synchronized JRTConnection setNewCurrentConnection() { List sources = getSources(); currentConnection = sources.get(ThreadLocalRandom.current().nextInt(0, sources.size())); - log.log(LogLevel.DEBUG, () -> "Choosing new connection: " + currentConnection); + log.log(FINE, () -> "Choosing new connection: " + currentConnection); return currentConnection; } diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java index 863e95c5625..2356b400343 100644 --- a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java +++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java @@ -8,12 +8,14 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.defaults.Defaults; import java.io.File; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.INFO; + /** * @author lesters */ @@ -47,7 +49,7 @@ public class UrlDownloader { Request request = new Request("frt.rpc.ping"); target.invokeSync(request, 5.0); if (! request.isError()) { - log.log(LogLevel.DEBUG, () -> "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + log.log(FINE, () -> "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); return; } else { target.close(); @@ -78,7 +80,7 @@ public class UrlDownloader { request.parameters().add(new StringValue(urlReference.value())); double rpcTimeout = Math.min(timeLeft, 60 * 60.0); - log.log(LogLevel.DEBUG, () -> "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); + log.log(FINE, () -> "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); target.invokeSync(request, rpcTimeout); if (request.checkReturnTypes("s")) { @@ -86,7 +88,7 @@ public class UrlDownloader { } else if (!request.isError()) { throw new RuntimeException("Invalid response: " + request.returnValues()); } else if (temporaryError(request)) { - log.log(LogLevel.INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); + log.log(INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); try { Thread.sleep(1000); } catch (InterruptedException e) { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java index 8c7c7f2703e..478dc6c45d6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ErrorCode; @@ -10,6 +9,8 @@ import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static java.util.logging.Level.INFO; + /** * Static utility methods for verifying common request properties. * @@ -23,31 +24,31 @@ public class RequestValidation { public static int validateRequest(JRTConfigRequest request) { ConfigKey key = request.getConfigKey(); if (!RequestValidation.verifyName(key.getName())) { - log.log(LogLevel.INFO, "Illegal name '" + key.getName() + "'"); + log.log(INFO, "Illegal name '" + key.getName() + "'"); return ErrorCode.ILLEGAL_NAME; } if (!RequestValidation.verifyNamespace(key.getNamespace())) { - log.log(LogLevel.INFO, "Illegal name space '" + key.getNamespace() + "'"); + log.log(INFO, "Illegal name space '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_NAME_SPACE; } if (!RequestValidation.verifyMd5(key.getMd5())) { - log.log(LogLevel.INFO, "Illegal md5 sum '" + key.getNamespace() + "'"); + log.log(INFO, "Illegal md5 sum '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_DEF_MD5; } if (!RequestValidation.verifyMd5(request.getRequestConfigMd5())) { - log.log(LogLevel.INFO, "Illegal config md5 '" + request.getRequestConfigMd5() + "'"); + log.log(INFO, "Illegal config md5 '" + request.getRequestConfigMd5() + "'"); return ErrorCode.ILLEGAL_CONFIG_MD5; } if (!RequestValidation.verifyGeneration(request.getRequestGeneration())) { - log.log(LogLevel.INFO, "Illegal generation '" + request.getRequestGeneration() + "'"); + log.log(INFO, "Illegal generation '" + request.getRequestGeneration() + "'"); return ErrorCode.ILLEGAL_GENERATION; } if (!RequestValidation.verifyTimeout(request.getTimeout())) { - log.log(LogLevel.INFO, "Illegal timeout '" + request.getTimeout() + "'"); + log.log(INFO, "Illegal timeout '" + request.getTimeout() + "'"); return ErrorCode.ILLEGAL_TIMEOUT; } if (!RequestValidation.verifyHostname(request.getClientHostName())) { - log.log(LogLevel.INFO, "Illegal client host name '" + request.getClientHostName() + "'"); + log.log(INFO, "Illegal client host name '" + request.getClientHostName() + "'"); return ErrorCode.ILLEGAL_CLIENT_HOSTNAME; } return 0; -- cgit v1.2.3 From 75ca432ec38e5cff9a58702fb4c42cf89dd0c844 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 23 Apr 2020 17:54:02 +0000 Subject: Priority has not been there for years.... --- .../docproc/jdisc/DocumentProcessingTask.java | 25 +--- .../com/yahoo/docproc/jdisc/RequestContext.java | 4 - .../jdisc/messagebus/MbusRequestContext.java | 15 --- ...cumentProcessingTaskPrioritizationTestCase.java | 131 --------------------- 4 files changed, 1 insertion(+), 174 deletions(-) delete mode 100644 docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingTaskPrioritizationTestCase.java diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index 4d887616093..d1d5256efe3 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -19,14 +19,13 @@ import java.util.Iterator; import java.util.List; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; /** * @author Einar M R Rosenvinge */ -public class DocumentProcessingTask implements Comparable, Runnable { +public class DocumentProcessingTask implements Runnable { private static Logger log = Logger.getLogger(DocumentProcessingTask.class.getName()); private final List processings = new ArrayList<>(); @@ -35,14 +34,11 @@ public class DocumentProcessingTask implements Comparable processing); void processingFailed(ErrorCode error, String msg); diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java index 187557791f3..2e63e2defff 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java @@ -47,7 +47,6 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { private final MbusRequest request; private final DocumentMessage requestMsg; private final ResponseHandler responseHandler; - private volatile int cachedApproxSize; // When spawning off new documents inside document processor, we do not want // throttling since this can lead to live locks. This is because the // document being processed is a resource and is then grabbing more resources of @@ -136,20 +135,6 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { ResponseDispatch.newInstance(response).dispatch(this); } - @Override - public int getApproxSize() { - if (cachedApproxSize > 0) { - return cachedApproxSize; - } - cachedApproxSize = requestMsg.getApproxSize(); - return cachedApproxSize; - } - - @Override - public int getPriority() { - return requestMsg.getPriority().getValue(); - } - @Override public URI getUri() { return request.getUri(); diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingTaskPrioritizationTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingTaskPrioritizationTestCase.java deleted file mode 100644 index f050b6c5450..00000000000 --- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingTaskPrioritizationTestCase.java +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.docproc.jdisc; - -import com.yahoo.docproc.Processing; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import org.junit.Test; - -import java.net.URI; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.PriorityBlockingQueue; - -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.CoreMatchers.sameInstance; -import static org.junit.Assert.assertThat; - -/** - * @author Einar M R Rosenvinge - */ -public class DocumentProcessingTaskPrioritizationTestCase { - - @Test - public void proritization() { - Queue queue = new PriorityBlockingQueue<>(); - - DocumentProcessingTask highest = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGHEST); - DocumentProcessingTask veryhigh = new TestDocumentProcessingTask(DocumentProtocol.Priority.VERY_HIGH); - DocumentProcessingTask high1 = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGH_1); - DocumentProcessingTask normal_1 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); - DocumentProcessingTask low_1 = new TestDocumentProcessingTask(DocumentProtocol.Priority.LOW_1); - DocumentProcessingTask verylow = new TestDocumentProcessingTask(DocumentProtocol.Priority.VERY_LOW); - DocumentProcessingTask lowest = new TestDocumentProcessingTask(DocumentProtocol.Priority.LOWEST); - - DocumentProcessingTask normal_2 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); - DocumentProcessingTask normal_3 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); - DocumentProcessingTask normal_4 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); - - DocumentProcessingTask highest_2 = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGHEST); - DocumentProcessingTask highest_3 = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGHEST); - - - queue.add(highest); - queue.add(veryhigh); - queue.add(high1); - queue.add(normal_1); - queue.add(low_1); - queue.add(verylow); - queue.add(lowest); - - queue.add(normal_2); - queue.add(normal_3); - queue.add(normal_4); - - queue.add(highest_2); - queue.add(highest_3); - - assertThat(queue.poll(), sameInstance(highest)); - assertThat(queue.poll(), sameInstance(highest_2)); - assertThat(queue.poll(), sameInstance(highest_3)); - assertThat(queue.poll(), sameInstance(veryhigh)); - assertThat(queue.poll(), sameInstance(high1)); - assertThat(queue.poll(), sameInstance(normal_1)); - assertThat(queue.poll(), sameInstance(normal_2)); - assertThat(queue.poll(), sameInstance(normal_3)); - assertThat(queue.poll(), sameInstance(normal_4)); - assertThat(queue.poll(), sameInstance(low_1)); - assertThat(queue.poll(), sameInstance(verylow)); - assertThat(queue.poll(), sameInstance(lowest)); - assertThat(queue.poll(), nullValue()); - } - - private class TestDocumentProcessingTask extends DocumentProcessingTask { - private TestDocumentProcessingTask(DocumentProtocol.Priority priority) { - super(new TestRequestContext(priority), null, null, null); - } - } - - private class TestRequestContext implements RequestContext { - private final DocumentProtocol.Priority priority; - - public TestRequestContext(DocumentProtocol.Priority priority) { - this.priority = priority; - } - - @Override - public List getProcessings() { - return null; - } - - @Override - public void skip() { - } - - @Override - public void processingDone(List processing) { - } - - @Override - public void processingFailed(ErrorCode error, String msg) { - } - - @Override - public void processingFailed(Exception exception) { - } - - @Override - public int getApproxSize() { - return 0; - } - - @Override - public int getPriority() { - return priority.getValue(); - } - - @Override - public boolean isProcessable() { - return true; - } - - @Override - public URI getUri() { - return null; - } - - @Override - public String getServiceName() { - return null; - } - } -} -- cgit v1.2.3 From 3a17d7e5852504539c022baa2a1cf96f2ec6061b Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 23 Apr 2020 20:30:33 +0200 Subject: Remove import of package which is not available or used --- .../java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java | 1 - .../yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java | 1 - 2 files changed, 2 deletions(-) diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 674571ff73e..9252c509261 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -3,7 +3,6 @@ package com.yahoo.searchlib.rankingexpression; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.yahoo.log.event.Collection; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java index f4b1b0ceee2..cf761aac2d3 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.transform; -import com.yahoo.log.event.Collection; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.Context; import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; -- cgit v1.2.3 From 6ef94983a16650f60db15485067e0a2bbf7bc420 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Thu, 23 Apr 2020 21:41:32 +0200 Subject: Revert "Use java.util.logging.Level instead of vespalog's LogLevel." --- config/pom.xml | 2 +- .../subscription/CfgConfigPayloadBuilder.java | 8 ++--- .../com/yahoo/config/subscription/ConfigDebug.java | 5 ++-- .../yahoo/config/subscription/ConfigSourceSet.java | 5 ++-- .../config/subscription/ConfigSubscriber.java | 11 +++---- .../subscription/impl/FileConfigSubscription.java | 17 +++++------ .../impl/GenericJRTConfigSubscription.java | 5 ++-- .../subscription/impl/JRTConfigRequester.java | 35 ++++++++++------------ .../subscription/impl/JRTConfigSubscription.java | 21 +++++++------ .../yahoo/vespa/config/ConfigPayloadApplier.java | 15 ++++------ .../com/yahoo/vespa/config/JRTConnectionPool.java | 5 ++-- .../java/com/yahoo/vespa/config/UrlDownloader.java | 10 +++---- .../vespa/config/protocol/RequestValidation.java | 17 +++++------ 13 files changed, 68 insertions(+), 88 deletions(-) diff --git a/config/pom.xml b/config/pom.xml index 6e4e26ed0f1..ad036c442d1 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -53,7 +53,7 @@ com.yahoo.vespa - defaults + vespalog ${project.version} diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java index 74d4d378e5d..474c9f7a4db 100644 --- a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java +++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java @@ -4,15 +4,13 @@ package com.yahoo.config.subscription; import com.yahoo.collections.Pair; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.StringNode; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; import java.util.ArrayList; import java.util.List; -import static java.util.logging.Level.FINE; -import static java.util.logging.Level.FINEST; - /** * Deserializes config payload (cfg format) to a ConfigPayload. * @@ -39,7 +37,7 @@ public class CfgConfigPayloadBuilder { parseLine(line, lineNum, payloadBuilder); lineNum++; } - log.log(FINEST, () -> "payload=" + payloadBuilder.toString()); + log.log(LogLevel.SPAM, () -> "payload=" + payloadBuilder.toString()); return payloadBuilder; } @@ -50,7 +48,7 @@ public class CfgConfigPayloadBuilder { String field = fieldAndValue.getFirst(); String value = fieldAndValue.getSecond(); if (field==null || value==null) { - log.log(FINE, () -> "Got field without value in line " + lineNum + ": " + line + ", skipping"); + log.log(LogLevel.DEBUG, () -> "Got field without value in line " + lineNum + ": " + line + ", skipping"); return; } field=field.trim(); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java index d279fc42251..3020f49b32e 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java @@ -2,17 +2,16 @@ package com.yahoo.config.subscription; import com.yahoo.config.ConfigInstance; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import java.util.logging.Logger; -import static java.util.logging.Level.INFO; - // Debug class that provides useful helper routines public class ConfigDebug { public static void logDebug(Logger logger, long timestamp, ConfigKey key, String logmessage) { if (key.getConfigId().matches(".*container.?\\d+.*") || key.getConfigId().matches(".*doc.api.*")) { - logger.log(INFO, timestamp + " " + key + " " + logmessage); + logger.log(LogLevel.INFO, timestamp + " " + key + " " + logmessage); } } diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java index e0605dce30d..7472439d6a4 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; +import com.yahoo.log.LogLevel; + import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; @@ -8,7 +10,6 @@ import java.util.List; import java.util.Set; import java.util.logging.Logger; -import static java.util.logging.Level.INFO; /** * An immutable set of connection endpoints, where each endpoint points to either a @@ -97,7 +98,7 @@ public class ConfigSourceSet implements ConfigSource { public static ConfigSourceSet createDefault() { String configSources = System.getenv("VESPA_CONFIG_SOURCES"); if (configSources != null) { - log.log(INFO, "Using config sources from VESPA_CONFIG_SOURCES: " + configSources); + log.log(LogLevel.INFO, "Using config sources from VESPA_CONFIG_SOURCES: " + configSources); return new ConfigSourceSet(checkSourcesSyntax(configSources)); } else { String[] def = {"tcp/localhost:" + System.getProperty("vespa.config.port", "19090")}; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index 814222989a1..3891d710fa3 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -5,6 +5,7 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.impl.ConfigSubscription; import com.yahoo.config.subscription.impl.JRTConfigRequester; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.TimingValues; import com.yahoo.yolean.Exceptions; @@ -13,12 +14,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; - import java.util.logging.Logger; -import static java.util.logging.Level.FINE; -import static java.util.logging.Level.SEVERE; -import static java.util.logging.Level.WARNING; import static java.util.stream.Collectors.toList; /** @@ -329,7 +326,7 @@ public class ConfigSubscriber implements AutoCloseable { h.subscription().close(); } closeRequesters(); - log.log(FINE, () -> "Config subscriber has been closed."); + log.log(LogLevel.DEBUG, () -> "Config subscriber has been closed."); } /** @@ -437,7 +434,7 @@ public class ConfigSubscriber implements AutoCloseable { if (handle.isChanged()) singleSubscriber.configure(handle.getConfig()); } } catch (Exception e) { - log.log(SEVERE, "Exception from config system, continuing config thread: " + Exceptions.toMessageString(e)); + log.log(LogLevel.ERROR, "Exception from config system, continuing config thread: " + Exceptions.toMessageString(e)); } } } @@ -480,7 +477,7 @@ public class ConfigSubscriber implements AutoCloseable { protected void finalize() throws Throwable { try { if (!isClosed()) { - log.log(WARNING, stackTraceAtConstruction, + log.log(LogLevel.WARNING, stackTraceAtConstruction, () -> String.format("%s: Closing subscription from finalizer() - close() has not been called (keys=%s)", super.toString(), subscriptionHandles.stream().map(handle -> handle.subscription().getKey().toString()).collect(toList()))); diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java index 365f6a4d4c5..60c920d1dfa 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java @@ -1,6 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.CfgConfigPayloadBuilder; @@ -9,12 +13,7 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.io.IOUtils; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; - -import static java.util.logging.Level.FINE; +import com.yahoo.log.LogLevel; /** * Subscription used when config id is file:... @@ -38,12 +37,12 @@ public class FileConfigSubscription extends ConfigSubs public boolean nextConfig(long timeout) { if (!file.exists() && !file.isFile()) throw new IllegalArgumentException("Not a file: "+file); if (checkReloaded()) { - log.log(FINE, "User forced config reload at " + System.currentTimeMillis()); + log.log(LogLevel.DEBUG, "User forced config reload at " + System.currentTimeMillis()); // User forced reload setConfigIfChanged(updateConfig()); ConfigState configState = getConfigState(); - log.log(FINE, "Config updated at " + System.currentTimeMillis() + ", changed: " + configState.isConfigChanged()); - log.log(FINE, "Config: " + configState.getConfig().toString()); + log.log(LogLevel.DEBUG, "Config updated at " + System.currentTimeMillis() + ", changed: " + configState.isConfigChanged()); + log.log(LogLevel.DEBUG, "Config: " + configState.getConfig().toString()); return true; } if (file.lastModified()!=ts) { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java index eec18b93e71..8ce3449fba5 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -5,14 +5,13 @@ import java.util.List; import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; -import static java.util.logging.Level.FINE; - /** * A JRT subscription which does not use the config class, but {@link com.yahoo.vespa.config.RawConfig} instead. * Used by config proxy. @@ -34,7 +33,7 @@ public class GenericJRTConfigSubscription extends JRTConfigSubscription "in setNewConfig, config=" + this.getConfigState().getConfig()); + log.log(LogLevel.DEBUG, () -> "in setNewConfig, config=" + this.getConfigState().getConfig()); } // This method is overridden because config needs to have its generation diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 6a43c397562..49c5dcd343c 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -6,6 +6,7 @@ import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Request; import com.yahoo.jrt.RequestWaiter; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.ErrorCode; @@ -23,12 +24,6 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; -import static java.util.logging.Level.FINE; -import static java.util.logging.Level.FINEST; -import static java.util.logging.Level.INFO; -import static java.util.logging.Level.SEVERE; -import static java.util.logging.Level.WARNING; - /** * This class fetches config payload using JRT, and acts as the callback target. * It uses the {@link JRTConfigSubscription} and {@link JRTClientConfigRequest} @@ -95,9 +90,9 @@ public class JRTConfigRequester implements RequestWaiter { if ( ! req.validateParameters()) throw new ConfigurationRuntimeException("Error in parameters for config request: " + req); double jrtClientTimeout = getClientTimeout(req); - log.log(FINE, () -> "Requesting config for " + sub + " on connection " + connection + log.log(LogLevel.DEBUG, () -> "Requesting config for " + sub + " on connection " + connection + " with client timeout " + jrtClientTimeout + - (log.isLoggable(FINEST) ? (",defcontent=" + req.getDefContent().asString()) : "")); + (log.isLoggable(LogLevel.SPAM) ? (",defcontent=" + req.getDefContent().asString()) : "")); connection.invokeAsync(req.getRequest(), jrtClientTimeout, this); } @@ -115,7 +110,7 @@ public class JRTConfigRequester implements RequestWaiter { sub.setException(e); } else { // Very unlikely - log.log(SEVERE, "Failed to get subscription object from JRT config callback: " + + log.log(Level.SEVERE, "Failed to get subscription object from JRT config callback: " + Exceptions.toMessageString(e)); } } @@ -123,11 +118,11 @@ public class JRTConfigRequester implements RequestWaiter { private void doHandle(JRTConfigSubscription sub, JRTClientConfigRequest jrtReq, Connection connection) { boolean validResponse = jrtReq.validateResponse(); - log.log(FINE, () -> "Request callback " + (validResponse ? "valid" : "invalid") + ". Req: " + jrtReq + "\nSpec: " + connection); + log.log(LogLevel.DEBUG, () -> "Request callback " + (validResponse ? "valid" : "invalid") + ". Req: " + jrtReq + "\nSpec: " + connection); if (sub.getState() == ConfigSubscription.State.CLOSED) return; // Avoid error messages etc. after closing Trace trace = jrtReq.getResponseTrace(); trace.trace(TRACELEVEL, "JRTConfigRequester.doHandle()"); - log.log(FINEST, () -> trace.toString()); + log.log(LogLevel.SPAM, () -> trace.toString()); if (validResponse) { handleOKRequest(jrtReq, sub, connection); } else { @@ -139,20 +134,20 @@ public class JRTConfigRequester implements RequestWaiter { private void logWhenErrorResponse(JRTClientConfigRequest jrtReq, Connection connection) { switch (jrtReq.errorCode()) { case com.yahoo.jrt.ErrorCode.CONNECTION: - log.log(FINE, () -> "Request callback failed: " + jrtReq.errorMessage() + + log.log(LogLevel.DEBUG, () -> "Request callback failed: " + jrtReq.errorMessage() + "\nConnection spec: " + connection); break; case ErrorCode.APPLICATION_NOT_LOADED: case ErrorCode.UNKNOWN_VESPA_VERSION: if (noApplicationWarningLogged.isBefore(Instant.now().minus(delayBetweenWarnings))) { - log.log(WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + + log.log(LogLevel.WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + ". Connection spec: " + connection.getAddress() + ", error message: " + jrtReq.errorMessage()); noApplicationWarningLogged = Instant.now(); } break; default: - log.log(WARNING, "Request callback failed. Req: " + jrtReq + "\nSpec: " + connection.getAddress() + + log.log(LogLevel.WARNING, "Request callback failed. Req: " + jrtReq + "\nSpec: " + connection.getAddress() + " . Req error message: " + jrtReq.errorMessage()); break; } @@ -162,7 +157,7 @@ public class JRTConfigRequester implements RequestWaiter { final boolean configured = (sub.getConfigState().getConfig() != null); if (configured) { // The subscription object has an "old" config, which is all we have to offer back now - log.log(INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); + log.log(LogLevel.INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); } ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode()); connectionPool.setError(connection, jrtReq.errorCode()); @@ -197,7 +192,7 @@ public class JRTConfigRequester implements RequestWaiter { Connection connection) { transientFailures++; if (suspendWarningLogged.isBefore(Instant.now().minus(delayBetweenWarnings))) { - log.log(INFO, "Connection to " + connection.getAddress() + + log.log(LogLevel.INFO, "Connection to " + connection.getAddress() + " failed or timed out, clients will keep existing config, will keep trying."); suspendWarningLogged = Instant.now(); } @@ -223,7 +218,7 @@ public class JRTConfigRequester implements RequestWaiter { if (sub.getState() != ConfigSubscription.State.OPEN) return; fatalFailures++; // The logging depends on whether we are configured or not. - Level logLevel = sub.getConfigState().getConfig() == null ? FINE : INFO; + Level logLevel = sub.getConfigState().getConfig() == null ? LogLevel.DEBUG : LogLevel.INFO; String logMessage = "Request for config " + jrtReq.getShortDescription() + "' failed with error code " + jrtReq.errorCode() + " (" + jrtReq.errorMessage() + "), scheduling new connect " + " in " + delay + " ms"; @@ -241,7 +236,7 @@ public class JRTConfigRequester implements RequestWaiter { noApplicationWarningLogged = Instant.MIN; connection.setSuccess(); sub.setLastCallBackOKTS(Instant.now()); - log.log(FINE, () -> "OK response received in handleOkRequest: " + jrtReq); + log.log(LogLevel.DEBUG, () -> "OK response received in handleOkRequest: " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { // We only want this latest generation to be in the queue, we do not preserve history in this system sub.getReqQueue().clear(); @@ -265,8 +260,8 @@ public class JRTConfigRequester implements RequestWaiter { private void scheduleNextRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription sub, long delay, long timeout) { long delayBeforeSendingRequest = (delay < 0) ? 0 : delay; JRTClientConfigRequest jrtReqNew = jrtReq.nextRequest(timeout); - log.log(FINEST, () -> timingValues.toString()); - log.log(FINE, () -> "Scheduling new request " + delayBeforeSendingRequest + " millis from now for " + jrtReqNew.getConfigKey()); + log.log(LogLevel.SPAM, () -> timingValues.toString()); + log.log(LogLevel.DEBUG, () -> "Scheduling new request " + delayBeforeSendingRequest + " millis from now for " + jrtReqNew.getConfigKey()); scheduler.schedule(new GetConfigTask(jrtReqNew, sub), delayBeforeSendingRequest, TimeUnit.MILLISECONDS); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index 44f6d65ee65..a94a135f9d8 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -1,26 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.ConfigInterruptedException; import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.JRTConnectionPool; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.CompressionType; import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; import com.yahoo.vespa.config.protocol.Payload; -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import static java.util.logging.Level.FINE; -import static java.util.logging.Level.INFO; - /** * A JRT config subscription uses one {@link JRTConfigRequester} to fetch config using Vespa RPC from a config source, typically proxy or server * @@ -91,7 +90,7 @@ public class JRTConfigSubscription extends ConfigSubsc // timed out, we know nothing new. return false; } - log.log(FINE, () -> "Polled queue and found config " + jrtReq); + log.log(LogLevel.DEBUG, () -> "Polled queue and found config " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { setInternalRedeploy(jrtReq.responseIsInternalRedeploy()); if (jrtReq.hasUpdatedConfig()) { @@ -193,7 +192,7 @@ public class JRTConfigSubscription extends ConfigSubsc @Override public void reload(long generation) { - log.log(FINE, "reload() is without effect on a JRTConfigSubscription."); + log.log(LogLevel.DEBUG, "reload() is without effect on a JRTConfigSubscription."); } void setLastCallBackOKTS(Instant lastCallBackOKTS) { @@ -205,7 +204,7 @@ public class JRTConfigSubscription extends ConfigSubsc static void printStatus(JRTClientConfigRequest request, String message) { final String name = request.getConfigKey().getName(); if (name.equals("components") || name.equals("chains")) { - log.log(INFO, message + ":" + name + ":" + ", request=" + request); + log.log(LogLevel.INFO, message + ":" + name + ":" + ", request=" + request); } } } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index b4df42c802e..134352736b6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -5,12 +5,13 @@ import com.yahoo.config.ConfigBuilder; import com.yahoo.config.ConfigInstance; import com.yahoo.config.FileReference; import com.yahoo.config.UrlReference; +import com.yahoo.log.LogLevel; +import com.yahoo.yolean.Exceptions; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Type; import com.yahoo.text.Utf8; -import com.yahoo.yolean.Exceptions; import java.io.File; import java.lang.reflect.Constructor; @@ -26,10 +27,6 @@ import java.util.Set; import java.util.Stack; import java.util.logging.Logger; -import static java.util.logging.Level.FINE; -import static java.util.logging.Level.FINEST; -import static java.util.logging.Level.INFO; - /** * A utility class that can be used to apply a payload to a config builder. * @@ -227,7 +224,7 @@ public class ConfigPayloadApplier { } catch (InvocationTargetException | IllegalAccessException e) { throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); } catch (NoSuchMethodException e) { - log.log(INFO, "Skipping unknown field " + methodName + " in " + rootBuilder); + log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + rootBuilder); } } @@ -282,7 +279,7 @@ public class ConfigPayloadApplier { } catch (InvocationTargetException | IllegalAccessException e) { throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); } catch (NoSuchMethodException e) { - log.log(INFO, "Skipping unknown field " + methodName + " in " + builder.getClass()); + log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + builder.getClass()); } } @@ -482,11 +479,11 @@ public class ConfigPayloadApplier { } private void debug(String message) { - log.log(FINE, () -> message); + log.log(LogLevel.DEBUG, () -> message); } private void trace(String message) { - log.log(FINEST, () -> message); + log.log(LogLevel.SPAM, () -> message); } private void printStack() { diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java index 2c1d00f295e..326c1287468 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; +import com.yahoo.log.LogLevel; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -12,8 +13,6 @@ import java.util.Map; import java.util.concurrent.ThreadLocalRandom; import java.util.logging.Logger; -import static java.util.logging.Level.FINE; - /** * A pool of JRT connections to a config source (either a config server or a config proxy). * The current connection is chosen randomly when calling {#link #setNewCurrentConnection} @@ -75,7 +74,7 @@ public class JRTConnectionPool implements ConnectionPool { public synchronized JRTConnection setNewCurrentConnection() { List sources = getSources(); currentConnection = sources.get(ThreadLocalRandom.current().nextInt(0, sources.size())); - log.log(FINE, () -> "Choosing new connection: " + currentConnection); + log.log(LogLevel.DEBUG, () -> "Choosing new connection: " + currentConnection); return currentConnection; } diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java index 2356b400343..863e95c5625 100644 --- a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java +++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java @@ -8,14 +8,12 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.defaults.Defaults; import java.io.File; import java.util.logging.Logger; -import static java.util.logging.Level.FINE; -import static java.util.logging.Level.INFO; - /** * @author lesters */ @@ -49,7 +47,7 @@ public class UrlDownloader { Request request = new Request("frt.rpc.ping"); target.invokeSync(request, 5.0); if (! request.isError()) { - log.log(FINE, () -> "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + log.log(LogLevel.DEBUG, () -> "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); return; } else { target.close(); @@ -80,7 +78,7 @@ public class UrlDownloader { request.parameters().add(new StringValue(urlReference.value())); double rpcTimeout = Math.min(timeLeft, 60 * 60.0); - log.log(FINE, () -> "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); + log.log(LogLevel.DEBUG, () -> "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); target.invokeSync(request, rpcTimeout); if (request.checkReturnTypes("s")) { @@ -88,7 +86,7 @@ public class UrlDownloader { } else if (!request.isError()) { throw new RuntimeException("Invalid response: " + request.returnValues()); } else if (temporaryError(request)) { - log.log(INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); + log.log(LogLevel.INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); try { Thread.sleep(1000); } catch (InterruptedException e) { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java index 478dc6c45d6..8c7c7f2703e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ErrorCode; @@ -9,8 +10,6 @@ import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static java.util.logging.Level.INFO; - /** * Static utility methods for verifying common request properties. * @@ -24,31 +23,31 @@ public class RequestValidation { public static int validateRequest(JRTConfigRequest request) { ConfigKey key = request.getConfigKey(); if (!RequestValidation.verifyName(key.getName())) { - log.log(INFO, "Illegal name '" + key.getName() + "'"); + log.log(LogLevel.INFO, "Illegal name '" + key.getName() + "'"); return ErrorCode.ILLEGAL_NAME; } if (!RequestValidation.verifyNamespace(key.getNamespace())) { - log.log(INFO, "Illegal name space '" + key.getNamespace() + "'"); + log.log(LogLevel.INFO, "Illegal name space '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_NAME_SPACE; } if (!RequestValidation.verifyMd5(key.getMd5())) { - log.log(INFO, "Illegal md5 sum '" + key.getNamespace() + "'"); + log.log(LogLevel.INFO, "Illegal md5 sum '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_DEF_MD5; } if (!RequestValidation.verifyMd5(request.getRequestConfigMd5())) { - log.log(INFO, "Illegal config md5 '" + request.getRequestConfigMd5() + "'"); + log.log(LogLevel.INFO, "Illegal config md5 '" + request.getRequestConfigMd5() + "'"); return ErrorCode.ILLEGAL_CONFIG_MD5; } if (!RequestValidation.verifyGeneration(request.getRequestGeneration())) { - log.log(INFO, "Illegal generation '" + request.getRequestGeneration() + "'"); + log.log(LogLevel.INFO, "Illegal generation '" + request.getRequestGeneration() + "'"); return ErrorCode.ILLEGAL_GENERATION; } if (!RequestValidation.verifyTimeout(request.getTimeout())) { - log.log(INFO, "Illegal timeout '" + request.getTimeout() + "'"); + log.log(LogLevel.INFO, "Illegal timeout '" + request.getTimeout() + "'"); return ErrorCode.ILLEGAL_TIMEOUT; } if (!RequestValidation.verifyHostname(request.getClientHostName())) { - log.log(INFO, "Illegal client host name '" + request.getClientHostName() + "'"); + log.log(LogLevel.INFO, "Illegal client host name '" + request.getClientHostName() + "'"); return ErrorCode.ILLEGAL_CLIENT_HOSTNAME; } return 0; -- cgit v1.2.3 From 284bb40ba71954361ceabd2fcc1257156a66bbdf Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 23 Apr 2020 22:00:43 +0200 Subject: Revert "Remove import of package which is not available or used" --- .../java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java | 1 + .../yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java | 1 + 2 files changed, 2 insertions(+) diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 9252c509261..674571ff73e 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -3,6 +3,7 @@ package com.yahoo.searchlib.rankingexpression; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.yahoo.log.event.Collection; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java index cf761aac2d3..f4b1b0ceee2 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.transform; +import com.yahoo.log.event.Collection; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.Context; import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; -- cgit v1.2.3 From afdeb2a86df8a471166252da1f178e5c528631b9 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 23 Apr 2020 21:52:40 +0200 Subject: nodes/v2/application responses WIP --- .../src/main/java/com/yahoo/config/provision/ApplicationId.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index f175af3e2f1..ef89c1d7525 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -47,6 +47,14 @@ public final class ApplicationId implements Comparable { return new Builder().tenant(parts[0]).applicationName(parts[1]).instanceName(parts[2]).build(); } + public static ApplicationId fromFullString(String idString) { + String[] parts = idString.split("."); + if (parts.length < 3) + throw new IllegalArgumentException("Application ids must be on the form tenant.application.instance, but was " + idString); + + return new Builder().tenant(parts[0]).applicationName(parts[1]).instanceName(parts[2]).build(); + } + @Override public int hashCode() { return stringValue.hashCode(); } -- cgit v1.2.3 From 6094d7c082d642530f1878f568be3e18e52b56a5 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 23 Apr 2020 21:51:07 +0200 Subject: nodes/v2/application responses WIP --- .../restapi/v2/ApplicationSerializer.java | 48 +++++++++++++++++ .../restapi/v2/NodeResourcesSerializer.java | 57 ++++++++++++++++++++ .../provision/restapi/v2/NodeSerializer.java | 45 ++-------------- .../provision/restapi/v2/NodesApiHandler.java | 62 ++++++++++++++++++---- .../hosted/provision/restapi/v2/NodesResponse.java | 13 +---- 5 files changed, 163 insertions(+), 62 deletions(-) create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ApplicationSerializer.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ApplicationSerializer.java new file mode 100644 index 00000000000..54f60651795 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ApplicationSerializer.java @@ -0,0 +1,48 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Cluster; + +import java.net.URI; +import java.util.Collection; + +/** + * Serializes application information for nodes/v2/application responses + */ +public class ApplicationSerializer { + + public static Slime toSlime(Application application, URI applicationUri) { + Slime slime = new Slime(); + toSlime(application, slime.setObject(), applicationUri); + return slime; + } + + private static void toSlime(Application application, Cursor object, URI applicationUri) { + object.setString("url", applicationUri.toString()); + object.setString("id", application.id().toFullString()); + clustersToSlime(application.clusters().values(), object.setObject("clusters")); + } + + private static void clustersToSlime(Collection clusters, Cursor clustersObject) { + clusters.forEach(cluster -> toSlime(cluster, clustersObject.setObject(cluster.id().value()))); + } + + private static void toSlime(Cluster cluster, Cursor clusterObject) { + toSlime(cluster.minResources(), clusterObject.setObject("min")); + toSlime(cluster.maxResources(), clusterObject.setObject("max")); + cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject("suggested"))); + cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject("target"))); + } + + private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { + clusterResourcesObject.setLong("nodes", resources.nodes()); + clusterResourcesObject.setLong("groups", resources.groups()); + NodeResourcesSerializer.toSlime(resources.nodeResources(), clusterResourcesObject.setObject("resources")); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java new file mode 100644 index 00000000000..10b3d6a4db8 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.config.provision.NodeResources; +import com.yahoo.slime.Cursor; + +/** + * @author bratseth + */ +public class NodeResourcesSerializer { + + static void toSlime(NodeResources resources, Cursor object) { + object.setDouble("vcpu", resources.vcpu()); + object.setDouble("memoryGb", resources.memoryGb()); + object.setDouble("diskGb", resources.diskGb()); + object.setDouble("bandwidthGbps", resources.bandwidthGbps()); + object.setString("diskSpeed", toString(resources.diskSpeed())); + object.setString("storageType", toString(resources.storageType())); + } + + public NodeResources.DiskSpeed diskSpeedFrom(String diskSpeed) { + switch (diskSpeed) { + case "fast": return NodeResources.DiskSpeed.fast; + case "slow": return NodeResources.DiskSpeed.slow; + case "any" : return NodeResources.DiskSpeed.any; + default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed + "'"); + } + } + + private static String toString(NodeResources.DiskSpeed diskSpeed) { + switch (diskSpeed) { + case fast : return "fast"; + case slow : return "slow"; + case any : return "any"; + default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed.name() + "'"); + } + } + + public NodeResources.StorageType storageTypeFrom(String storageType) { + switch (storageType) { + case "local" : return NodeResources.StorageType.local; + case "remote": return NodeResources.StorageType.remote; + case "any" : return NodeResources.StorageType.any; + default: throw new IllegalArgumentException("Unknown storage type '" + storageType + "'"); + } + } + + private static String toString(NodeResources.StorageType storageType) { + switch (storageType) { + case remote : return "remote"; + case local : return "local"; + case any : return "any"; + default: throw new IllegalArgumentException("Unknown storage type '" + storageType.name() + "'"); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java index 492994c2f0e..67f3b293de8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.slime.Cursor; import com.yahoo.vespa.hosted.provision.Node; /** @@ -12,7 +13,7 @@ import com.yahoo.vespa.hosted.provision.Node; */ public class NodeSerializer { - public Node.State stateFrom(String state) { + public static Node.State stateFrom(String state) { switch (state) { case "active": return Node.State.active; case "dirty": return Node.State.dirty; @@ -27,7 +28,7 @@ public class NodeSerializer { } } - public String toString(Node.State state) { + public static String toString(Node.State state) { switch (state) { case active: return "active"; case dirty: return "dirty"; @@ -42,7 +43,7 @@ public class NodeSerializer { } } - public NodeType typeFrom(String nodeType) { + public static NodeType typeFrom(String nodeType) { switch (nodeType) { case "tenant": return NodeType.tenant; case "host": return NodeType.host; @@ -57,7 +58,7 @@ public class NodeSerializer { } } - public String toString(NodeType type) { + public static String toString(NodeType type) { switch (type) { case tenant: return "tenant"; case host: return "host"; @@ -72,40 +73,4 @@ public class NodeSerializer { } } - public NodeResources.DiskSpeed diskSpeedFrom(String diskSpeed) { - switch (diskSpeed) { - case "fast": return NodeResources.DiskSpeed.fast; - case "slow": return NodeResources.DiskSpeed.slow; - case "any" : return NodeResources.DiskSpeed.any; - default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed + "'"); - } - } - - public String toString(NodeResources.DiskSpeed diskSpeed) { - switch (diskSpeed) { - case fast : return "fast"; - case slow : return "slow"; - case any : return "any"; - default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed.name() + "'"); - } - } - - public NodeResources.StorageType storageTypeFrom(String storageType) { - switch (storageType) { - case "local" : return NodeResources.StorageType.local; - case "remote": return NodeResources.StorageType.remote; - case "any" : return NodeResources.StorageType.any; - default: throw new IllegalArgumentException("Unknown storage type '" + storageType + "'"); - } - } - - public String toString(NodeResources.StorageType storageType) { - switch (storageType) { - case remote : return "remote"; - case local : return "local"; - case any : return "any"; - default: throw new IllegalArgumentException("Unknown storage type '" + storageType.name() + "'"); - } - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 809e4200e7e..9c1b1e03b11 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; @@ -17,7 +18,9 @@ import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; @@ -25,6 +28,7 @@ import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; @@ -42,6 +46,8 @@ import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -100,17 +106,20 @@ public class NodesApiHandler extends LoggingRequestHandler { } private HttpResponse handleGET(HttpRequest request) { - String path = request.getUri().getPath(); - if (path.equals( "/nodes/v2/")) return new ResourceResponse(request.getUri(), "state", "node", "command", "maintenance", "upgrade"); - if (path.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository); - if (path.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository); - if (path.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, orchestrator, nodeRepository); - if (path.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, orchestrator, nodeRepository); - if (path.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); - if (path.equals( "/nodes/v2/command/")) return new ResourceResponse(request.getUri(), "restart", "reboot"); - if (path.equals( "/nodes/v2/maintenance/")) return new JobsResponse(nodeRepository.jobControl()); - if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(nodeRepository.infrastructureVersions(), nodeRepository.osVersions(), nodeRepository.dockerImages()); - if (path.startsWith("/nodes/v2/capacity")) return new HostCapacityResponse(nodeRepository, request); + Path path = new Path(request.getUri()); + String pathS = request.getUri().toString(); + if (pathS.equals( "/nodes/v2/")) return new ResourceResponse(request.getUri(), "state", "node", "command", "maintenance", "upgrade"); + if (pathS.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository); + if (pathS.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository); + if (pathS.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, orchestrator, nodeRepository); + if (pathS.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, orchestrator, nodeRepository); + if (pathS.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); + if (pathS.equals( "/nodes/v2/command/")) return new ResourceResponse(request.getUri(), "restart", "reboot"); + if (pathS.equals( "/nodes/v2/maintenance/")) return new JobsResponse(nodeRepository.jobControl()); + if (pathS.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(nodeRepository.infrastructureVersions(), nodeRepository.osVersions(), nodeRepository.dockerImages()); + if (pathS.startsWith("/nodes/v2/capacity")) return new HostCapacityResponse(nodeRepository, request); + if (path.matches("/nodes/v2/application")) return applicationList(request.getUri()); + if (path.matches("/nodes/v2/application/{applicationId}")) return application(path.get("applicationId"), request.getUri()); throw new NotFoundException("Nothing at path '" + path + "'"); } @@ -394,4 +403,35 @@ public class NodesApiHandler extends LoggingRequestHandler { return nodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); } + private HttpResponse applicationList(URI uri) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor applications = root.setArray("applications"); + for (ApplicationId id : nodeRepository.applications().ids()) { + Cursor application = applications.addObject(); + application.setString("url", withPath("nodes/v2/applications/" + id.toFullString(), uri).toString()); + application.setString("id", id.toFullString()); + } + return new SlimeJsonResponse(slime); + } + + private HttpResponse application(String id, URI uri) { + Optional application = nodeRepository.applications().get(ApplicationId.fromFullString(id)); + if (application.isEmpty()) + return ErrorResponse.notFoundError("No application '" + id + "'"); + Slime slime = ApplicationSerializer.toSlime(application.get(), + withPath("nodes/v2/applications/" + id, uri)); + return new SlimeJsonResponse(slime); + } + + /** Returns a copy of the given URI with the host and port from the given URI and the path set to the given path */ + private URI withPath(String newPath, URI uri) { + try { + return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), newPath, null, null); + } + catch (URISyntaxException e) { + throw new RuntimeException("Will not happen", e); + } + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index a4412a502aa..fbb9b207c32 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -148,7 +148,7 @@ class NodesResponse extends HttpResponse { node.reservedTo().ifPresent(reservedTo -> object.setString("reservedTo", reservedTo.value())); if (node.flavor().isConfigured()) object.setDouble("cpuCores", node.flavor().getMinCpuCores()); - toSlime(node.flavor().resources(), object.setObject("resources")); + NodeResourcesSerializer.toSlime(node.flavor().resources(), object.setObject("resources")); if (node.flavor().cost() > 0) object.setLong("cost", node.flavor().cost()); object.setString("environment", node.flavor().getType().name()); @@ -160,7 +160,7 @@ class NodesResponse extends HttpResponse { object.setString("wantedDockerImage", allocation.membership().cluster().dockerImage() .orElseGet(() -> nodeRepository.dockerImage(node).withTag(allocation.membership().cluster().vespaVersion()).asString())); object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); - toSlime(allocation.requestedResources(), object.setObject("requestedResources")); + NodeResourcesSerializer.toSlime(allocation.requestedResources(), object.setObject("requestedResources")); allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); orchestrator.apply(new HostName(node.hostname())) .ifPresent(info -> { @@ -210,15 +210,6 @@ class NodesResponse extends HttpResponse { } } - private void toSlime(NodeResources resources, Cursor object) { - object.setDouble("vcpu", resources.vcpu()); - object.setDouble("memoryGb", resources.memoryGb()); - object.setDouble("diskGb", resources.diskGb()); - object.setDouble("bandwidthGbps", resources.bandwidthGbps()); - object.setString("diskSpeed", serializer.toString(resources.diskSpeed())); - object.setString("storageType", serializer.toString(resources.storageType())); - } - // Hack: For non-docker nodes, return current docker image as default prefix + current Vespa version // TODO: Remove current + wanted docker image from response for non-docker types private Optional currentDockerImage(Node node) { -- cgit v1.2.3 From 786498a263616b0c61dacd4599ecff11d32cd865 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 23 Apr 2020 22:06:19 +0000 Subject: GC unused code --- .../com/yahoo/feedapi/DocprocMessageProcessor.java | 72 ---------------------- 1 file changed, 72 deletions(-) delete mode 100755 vespaclient-core/src/main/java/com/yahoo/feedapi/DocprocMessageProcessor.java diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/DocprocMessageProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/DocprocMessageProcessor.java deleted file mode 100755 index c6974cff5c1..00000000000 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/DocprocMessageProcessor.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedapi; - -import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.docproc.CallStack; -import com.yahoo.docproc.DocprocService; -import com.yahoo.docproc.DocumentProcessor; -import com.yahoo.docproc.Processing; -import com.yahoo.document.DocumentOperation; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.routing.Route; - -import java.util.ArrayList; -import java.util.List; - -public class DocprocMessageProcessor implements MessageProcessor { - private final DocprocService docproc; - private final ComponentRegistry docprocServiceRegistry; - - public DocprocMessageProcessor(DocprocService docproc, ComponentRegistry docprocServiceRegistry) { - this.docproc = docproc; - this.docprocServiceRegistry = docprocServiceRegistry; - } - - @Override - public void process(Message m) { - try { - List documentBases = new ArrayList<>(1); - - if (m.getType() == DocumentProtocol.MESSAGE_PUTDOCUMENT) { - documentBases.add(((PutDocumentMessage) m).getDocumentPut()); - } else if (m.getType() == DocumentProtocol.MESSAGE_UPDATEDOCUMENT) { - documentBases.add(((UpdateDocumentMessage) m).getDocumentUpdate()); - } else if (m.getType() == DocumentProtocol.MESSAGE_REMOVEDOCUMENT) { - documentBases.add(((RemoveDocumentMessage) m).getDocumentRemove()); - } - - if (docproc != null) { - processDocumentOperations(documentBases, m); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private void processDocumentOperations(List documentOperations, Message m) throws Exception { - Processing processing = Processing.createProcessingFromDocumentOperations(docproc.getName(), documentOperations, new CallStack(docproc.getCallStack())); - processing.setServiceName(docproc.getName()); - processing.setDocprocServiceRegistry(docprocServiceRegistry); - processing.setVariable("route", m.getRoute()); - processing.setVariable("timeout", m.getTimeRemaining()); - - DocumentProcessor.Progress progress = docproc.getExecutor().process(processing); - while (DocumentProcessor.Progress.LATER.equals(progress)) { - Thread.sleep(50); - progress = docproc.getExecutor().process(processing); - } - - if (progress == DocumentProcessor.Progress.FAILED - || progress == DocumentProcessor.Progress.PERMANENT_FAILURE) { - throw new RuntimeException("Processing of " + documentOperations + " failed: " + progress + "."); - } - - m.setRoute((Route) processing.getVariable("route")); - m.setTimeRemaining((Long) processing.getVariable("timeout")); - } - -} -- cgit v1.2.3 From 2319fe6f00fe0e1891f26f919de3ec9796752a2c Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 24 Apr 2020 09:51:15 +0200 Subject: Add docker image repository to node details --- .../controller/deployment/InternalStepRunner.java | 13 +++++++++++-- .../responses/dev-us-east-1-log-first-part.json | 2 +- .../application/responses/staging-test-log.json | 8 ++++---- .../application/responses/system-test-details.json | 20 ++++++++++---------- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index c49c56c4e82..353b82ede49 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -526,8 +526,8 @@ public class InternalStepRunner implements StepRunner { private Stream nodeDetails(NodeWithServices node, boolean printAllServices) { return Stream.concat(Stream.of(node.node().hostname() + ": " + humanize(node.node().serviceState()) + (node.node().suspendedSince().map(since -> " since " + since).orElse("")), - "--- platform " + node.node().wantedVersion() + (node.needsPlatformUpgrade() - ? " <-- " + (node.node().currentVersion().isEmpty() ? "not booted" : node.node().currentVersion()) + "--- platform " + wantedPlatform(node.node()) + (node.needsPlatformUpgrade() + ? " <-- " + currentPlatform(node.node()) : "") + (node.needsOsUpgrade() && node.isAllowedDown() ? ", upgrading OS (" + node.node().wantedOsVersion() + " <-- " + node.node().currentOsVersion() + ")" @@ -548,6 +548,15 @@ public class InternalStepRunner implements StepRunner { : " has config generation " + service.currentGeneration() + ", wanted is " + node.wantedConfigGeneration()))); } + + private String wantedPlatform(Node node) { + return node.wantedDockerImage().repository() + ":" + node.wantedVersion(); + } + + private String currentPlatform(Node node) { + return node.wantedDockerImage().repository() + ":" + node.wantedVersion(); + } + private String humanize(Node.ServiceState state) { switch (state) { case allowedDown: return "allowed to be DOWN"; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json index 3a54ac70b0d..50799bcefa2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json @@ -38,7 +38,7 @@ { "at": 0, "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index ba51471d467..a941b824b53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -28,7 +28,7 @@ { "at": 7303000, "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": 7303000, @@ -43,7 +43,7 @@ { "at": 7303000, "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": 7303000, @@ -58,7 +58,7 @@ { "at": 7303000, "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": 7303000, @@ -102,7 +102,7 @@ { "at": 7303000, "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": 7303000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index 489d6a11b6a..ec45ae02487 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -33,7 +33,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -48,7 +48,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -63,7 +63,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -78,7 +78,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -122,7 +122,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -137,7 +137,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -152,7 +152,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -167,7 +167,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -182,7 +182,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -197,7 +197,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", -- cgit v1.2.3 From 74368e954fb257799d38f2b7d4982ae07c58948c Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 24 Apr 2020 10:27:28 +0200 Subject: Add nodes/v2/application paths --- .../provision/restapi/v2/NodeResourcesSerializer.java | 4 ++-- .../vespa/hosted/provision/restapi/v2/NodesApiHandler.java | 13 ++++++------- .../vespa/hosted/provision/restapi/v2/NodesResponse.java | 12 +++++------- .../hosted/provision/restapi/v2/NodeSerializerTest.java | 6 ++---- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java index 10b3d6a4db8..0d0c9305a92 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java @@ -18,7 +18,7 @@ public class NodeResourcesSerializer { object.setString("storageType", toString(resources.storageType())); } - public NodeResources.DiskSpeed diskSpeedFrom(String diskSpeed) { + public static NodeResources.DiskSpeed diskSpeedFrom(String diskSpeed) { switch (diskSpeed) { case "fast": return NodeResources.DiskSpeed.fast; case "slow": return NodeResources.DiskSpeed.slow; @@ -36,7 +36,7 @@ public class NodeResourcesSerializer { } } - public NodeResources.StorageType storageTypeFrom(String storageType) { + public static NodeResources.StorageType storageTypeFrom(String storageType) { switch (storageType) { case "local" : return NodeResources.StorageType.local; case "remote": return NodeResources.StorageType.remote; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 9c1b1e03b11..e275d978e4e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -70,7 +70,6 @@ public class NodesApiHandler extends LoggingRequestHandler { private final Orchestrator orchestrator; private final NodeRepository nodeRepository; private final NodeFlavors nodeFlavors; - private final NodeSerializer serializer = new NodeSerializer(); @Inject public NodesApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator, @@ -107,7 +106,7 @@ public class NodesApiHandler extends LoggingRequestHandler { private HttpResponse handleGET(HttpRequest request) { Path path = new Path(request.getUri()); - String pathS = request.getUri().toString(); + String pathS = request.getUri().getPath(); if (pathS.equals( "/nodes/v2/")) return new ResourceResponse(request.getUri(), "state", "node", "command", "maintenance", "upgrade"); if (pathS.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository); if (pathS.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository); @@ -263,8 +262,8 @@ public class NodesApiHandler extends LoggingRequestHandler { requiredField(resourcesInspector, "memoryGb", Inspector::asDouble), requiredField(resourcesInspector, "diskGb", Inspector::asDouble), requiredField(resourcesInspector, "bandwidthGbps", Inspector::asDouble), - optionalString(resourcesInspector.field("diskSpeed")).map(serializer::diskSpeedFrom).orElse(NodeResources.DiskSpeed.getDefault()), - optionalString(resourcesInspector.field("storageType")).map(serializer::storageTypeFrom).orElse(NodeResources.StorageType.getDefault()))); + optionalString(resourcesInspector.field("diskSpeed")).map(NodeResourcesSerializer::diskSpeedFrom).orElse(NodeResources.DiskSpeed.getDefault()), + optionalString(resourcesInspector.field("storageType")).map(NodeResourcesSerializer::storageTypeFrom).orElse(NodeResources.StorageType.getDefault()))); } Flavor flavor = nodeFlavors.getFlavorOrThrow(flavorInspector.asString()); @@ -278,9 +277,9 @@ public class NodesApiHandler extends LoggingRequestHandler { if (resourcesInspector.field("bandwidthGbps").valid()) flavor = flavor.with(flavor.resources().withBandwidthGbps(resourcesInspector.field("bandwidthGbps").asDouble())); if (resourcesInspector.field("diskSpeed").valid()) - flavor = flavor.with(flavor.resources().with(serializer.diskSpeedFrom(resourcesInspector.field("diskSpeed").asString()))); + flavor = flavor.with(flavor.resources().with(NodeResourcesSerializer.diskSpeedFrom(resourcesInspector.field("diskSpeed").asString()))); if (resourcesInspector.field("storageType").valid()) - flavor = flavor.with(flavor.resources().with(serializer.storageTypeFrom(resourcesInspector.field("storageType").asString()))); + flavor = flavor.with(flavor.resources().with(NodeResourcesSerializer.storageTypeFrom(resourcesInspector.field("storageType").asString()))); } return flavor; } @@ -293,7 +292,7 @@ public class NodesApiHandler extends LoggingRequestHandler { private NodeType nodeTypeFromSlime(Inspector object) { if (! object.valid()) return NodeType.tenant; // default - return serializer.typeFrom(object.asString()); + return NodeSerializer.typeFrom(object.asString()); } private Optional reservedToFromSlime(Inspector object) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index fbb9b207c32..c6b92d0a7f5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; @@ -48,7 +47,6 @@ class NodesResponse extends HttpResponse { private final Function> orchestrator; private final NodeRepository nodeRepository; private final Slime slime; - private final NodeSerializer serializer = new NodeSerializer(); public NodesResponse(ResponseType responseType, HttpRequest request, Orchestrator orchestrator, NodeRepository nodeRepository) { @@ -65,7 +63,7 @@ class NodesResponse extends HttpResponse { switch (responseType) { case nodeList: nodesToSlime(root); break; case stateList : statesToSlime(root); break; - case nodesInStateList: nodesToSlime(serializer.stateFrom(lastElement(parentUrl)), root); break; + case nodesInStateList: nodesToSlime(NodeSerializer.stateFrom(lastElement(parentUrl)), root); break; case singleNode : nodeToSlime(lastElement(parentUrl), root); break; default: throw new IllegalArgumentException(); } @@ -97,11 +95,11 @@ class NodesResponse extends HttpResponse { private void statesToSlime(Cursor root) { Cursor states = root.setObject("states"); for (Node.State state : Node.State.values()) - toSlime(state, states.setObject(serializer.toString(state))); + toSlime(state, states.setObject(NodeSerializer.toString(state))); } private void toSlime(Node.State state, Cursor object) { - object.setString("url", parentUrl + serializer.toString(state)); + object.setString("url", parentUrl + NodeSerializer.toString(state)); if (recursive) nodesToSlime(state, object); } @@ -136,10 +134,10 @@ class NodesResponse extends HttpResponse { object.setString("url", nodeParentUrl + node.hostname()); if ( ! allFields) return; object.setString("id", node.hostname()); - object.setString("state", serializer.toString(node.state())); + object.setString("state", NodeSerializer.toString(node.state())); object.setString("type", node.type().name()); object.setString("hostname", node.hostname()); - object.setString("type", serializer.toString(node.type())); + object.setString("type", NodeSerializer.toString(node.type())); if (node.parentHostname().isPresent()) { object.setString("parentHostname", node.parentHostname().get()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java index 05b23addddf..ed2aae64b5f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java @@ -13,19 +13,17 @@ import static org.junit.Assert.assertEquals; */ public class NodeSerializerTest { - private final NodeSerializer serializer = new NodeSerializer(); - @Test public void serialize_node_types() { for (NodeType t : NodeType.values()) { - assertEquals(t, serializer.typeFrom(serializer.toString(t))); + assertEquals(t, NodeSerializer.typeFrom(NodeSerializer.toString(t))); } } @Test public void serialize_node_states() { for (Node.State s : Node.State.values()) { - assertEquals(s, serializer.stateFrom(serializer.toString(s))); + assertEquals(s, NodeSerializer.stateFrom(NodeSerializer.toString(s))); } } -- cgit v1.2.3 From 976f28680df424bf028eb12a3f413049c17bf098 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 24 Apr 2020 08:34:55 +0000 Subject: add more tracing and debug logging of stemming --- .../com/yahoo/prelude/querytransform/StemmingSearcher.java | 13 +++++++++++++ .../java/com/yahoo/language/opennlp/OpenNlpLinguistics.java | 4 ++++ .../java/com/yahoo/language/opennlp/OpenNlpTokenizer.java | 8 ++++++++ .../java/com/yahoo/language/opennlp/OptimaizeDetector.java | 4 ++++ .../java/com/yahoo/language/simple/SimpleTokenizer.java | 10 +++++++++- 5 files changed, 38 insertions(+), 1 deletion(-) diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 9a9044def2d..9b846d9f0ae 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -98,6 +98,7 @@ public class StemmingSearcher extends Searcher { context.language = language; context.indexFacts = indexFacts; context.reverseConnectivity = createReverseConnectivities(q.getModel().getQueryTree().getRoot()); + q.trace("Stemming with language="+language, 3); return scan(q.getModel().getQueryTree().getRoot(), context); } @@ -183,9 +184,20 @@ public class StemmingSearcher extends Searcher { Substring substring = getOffsets(current); if (segments.size() == 1) { + getLogger().log(LogLevel.DEBUG, "Stem '"+current.stringValue()+"' mode "+index.getStemMode() + +" and language '"+context.language+"' -> '"+segments.get(0)+"'"); TaggableItem w = singleWordSegment(current, segments.get(0), index, substring, context.insidePhrase); setMetaData(current, context.reverseConnectivity, w); return (Item) w; + } else if (getLogger().isLoggable(LogLevel.DEBUG)) { + var buf = new StringBuilder(); + buf.append("Stem '").append(current.stringValue()); + buf.append("' mode ").append(index.getStemMode()); + buf.append(" and language '").append(context.language).append("' ->"); + for (StemList segment : segments) { + buf.append(" '").append(segment).append("'"); + } + getLogger().log(LogLevel.DEBUG, buf.toString()); } if (context.isCJK) @@ -194,6 +206,7 @@ public class StemmingSearcher extends Searcher { composite = chooseComposite(current, ((Item) current).getParent(), indexName); for (StemList segment : segments) { + getLogger().log(LogLevel.DEBUG, "Stem to multiple segments '"+segment+"'"); TaggableItem w = singleWordSegment(current, segment, index, substring, context.insidePhrase); if (composite instanceof AndSegmentItem) { diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index 1c7c71c00b6..0837b25c151 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -6,6 +6,8 @@ import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.simple.SimpleDetector; import com.yahoo.language.simple.SimpleLinguistics; +import java.util.logging.Logger; +import java.util.logging.Level; /** * Returns a linguistics implementation based on OpenNlp, @@ -13,6 +15,7 @@ import com.yahoo.language.simple.SimpleLinguistics; */ public class OpenNlpLinguistics extends SimpleLinguistics { + private static final Logger log = Logger.getLogger(OpenNlpLinguistics.class.getName()); private final Detector detector; public OpenNlpLinguistics() { @@ -26,6 +29,7 @@ public class OpenNlpLinguistics extends SimpleLinguistics { public OpenNlpLinguistics(boolean enableOptimaize) { this(enableOptimaize ? new OptimaizeDetector() : new SimpleDetector()); + log.log(Level.FINE, "using "+(enableOptimaize ? "Optimaize" : "Simple")+" detector"); } private OpenNlpLinguistics(Detector detector) { diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java index 07349811bd4..0ebd4e0f638 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -11,9 +11,12 @@ import opennlp.tools.stemmer.snowball.SnowballStemmer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.logging.Logger; +import java.util.logging.Level; public class OpenNlpTokenizer implements Tokenizer { private final static int SPACE_CODE = 32; + private static final Logger log = Logger.getLogger(OpenNlpTokenizer.class.getName()); private final Normalizer normalizer; private final Transformer transformer; private final SimpleTokenizer simpleTokenizer; @@ -57,6 +60,7 @@ public class OpenNlpTokenizer implements Tokenizer { } private Stemmer getStemmerForLanguage(Language language, StemMode stemMode) { + log.log(Level.FINEST, "getStemmerForLanguage '"+language+"' mode: "+stemMode); if (language == null || Language.ENGLISH.equals(language) || StemMode.NONE.equals(stemMode)) { return null; } @@ -120,13 +124,17 @@ public class OpenNlpTokenizer implements Tokenizer { private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, Stemmer stemmer) { + log.log(Level.FINEST, "processToken '"+token+"'"); token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); if (stemMode != StemMode.NONE) { + String oldToken = token; token = doStemming(token, stemmer); + log.log(Level.FINEST, "stem '"+oldToken+"' to '"+token+"'"); } + log.log(Level.FINEST, "processed token is: "+token); return token; } diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java index ef1d6c966ac..a42c9f0504e 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java @@ -22,6 +22,8 @@ import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.util.List; import java.util.Locale; +import java.util.logging.Logger; +import java.util.logging.Level; /** * Detects the language of some sample text using SimpleDetector for CJK and Optimaize otherwise. @@ -33,6 +35,7 @@ public class OptimaizeDetector implements Detector { static private Object initGuard = new Object(); static private TextObjectFactory textObjectFactory = null; static private LanguageDetector languageDetector = null; + static private final Logger log = Logger.getLogger(OptimaizeDetector.class.getName()); static private void initOptimaize() { synchronized (initGuard) { @@ -96,6 +99,7 @@ public class OptimaizeDetector implements Detector { private static Language guessLanguageUsingOptimaize(String input) { Optional result = languageDetector.detect(textObjectFactory.forText(input)); if ( ! result.isPresent()) return Language.UNKNOWN; + log.log(Level.FINE, "guessing language "+result.get()+" from input: "+input); return Language.fromLocale(new Locale(result.get().getLanguage())); } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index 068fc0126d7..a8470d86869 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -9,6 +9,8 @@ import com.yahoo.language.simple.kstem.KStemmer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.logging.Logger; +import java.util.logging.Level; /** *

A tokenizer which splits on whitespace, normalizes and transforms using the given implementations @@ -25,6 +27,7 @@ public class SimpleTokenizer implements Tokenizer { private final Normalizer normalizer; private final Transformer transformer; private final KStemmer stemmer = new KStemmer(); + private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName()); public SimpleTokenizer() { this(new SimpleNormalizer(), new SimpleTransformer()); @@ -64,12 +67,17 @@ public class SimpleTokenizer implements Tokenizer { } private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) { + log.log(Level.FINEST, "processToken '"+token+"'"); token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); - if (stemMode != StemMode.NONE) + if (stemMode != StemMode.NONE) { + String oldToken = token; token = stemmer.stem(token); + log.log(Level.FINEST, "stem '"+oldToken+"' to '"+token+"'"); + } + log.log(Level.FINEST, "processed token is: "+token); return token; } -- cgit v1.2.3 From e83700c6f617f5fbcb78be9f7a1289523aee987d Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 24 Apr 2020 10:37:03 +0200 Subject: Use JimFS instead --- .../hosted/node/admin/task/util/file/FileFinderTest.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java index 197077159d1..8fff2141252 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java @@ -2,15 +2,15 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.runners.Enclosed; -import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -40,8 +40,8 @@ import static org.mockito.Mockito.when; public class FileFinderTest { public static class GeneralLogicTests { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); + + private final FileSystem fileSystem = TestFileSystem.create(); @Test public void all_files_non_recursive() { @@ -111,6 +111,7 @@ public class FileFinderTest { @Before public void setup() throws IOException { Path root = testRoot(); + Files.createDirectories(root); Files.createFile(root.resolve("file-1.json")); Files.createFile(root.resolve("test.json")); @@ -127,7 +128,7 @@ public class FileFinderTest { } private Path testRoot() { - return folder.getRoot().toPath(); + return fileSystem.getPath("/file-finder"); } private void assertFileHelper(FileFinder fileFinder, Set expectedList, Set expectedContentsAfterDelete) { -- cgit v1.2.3 From 59684f45818f16dfffeacec1a9dcb3bd7aa80ed0 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 24 Apr 2020 10:57:03 +0200 Subject: Implement FileFinder::prune --- .../node/admin/task/util/file/FileFinder.java | 26 ++++++++++++++++++++++ .../node/admin/task/util/file/FileFinderTest.java | 21 +++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java index a41281d0479..f9a369aab27 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java @@ -14,6 +14,8 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -35,6 +37,7 @@ public class FileFinder { private static final Logger logger = Logger.getLogger(FileFinder.class.getName()); private final Path basePath; + private final Set pruned = new HashSet<>(); private Predicate matcher; private int maxDepth = Integer.MAX_VALUE; @@ -71,6 +74,27 @@ public class FileFinder { return this; } + /** + * Path for which whole directory tree will be skipped, including the path itself. + * The path must be under {@code basePath} or be relative to {@code basePath}. + */ + public FileFinder prune(Path path) { + if (!path.isAbsolute()) + path = basePath.resolve(path); + + if (!path.startsWith(basePath)) + throw new IllegalArgumentException("Prune path " + path + " is not under base path " + basePath); + + this.pruned.add(path); + return this; + } + + /** Convenience method for pruning multiple paths, see {@link #prune(Path)}. */ + public FileFinder prune(Collection paths) { + paths.forEach(this::prune); + return this; + } + /** * Maximum depth (relative to basePath) where contents should be matched with the given filters. * Default is unlimited. @@ -145,6 +169,8 @@ public class FileFinder { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + if (pruned.contains(dir)) return FileVisitResult.SKIP_SUBTREE; + currentLevel++; FileAttributes attributes = new FileAttributes(dir, attrs); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java index 8fff2141252..26897839b32 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java @@ -60,6 +60,27 @@ public class FileFinderTest { of("test", "test/subdir-1", "test/subdir-2")); } + @Test + public void all_files_recursive_with_prune_relative() { + assertFileHelper(FileFinder.files(testRoot()).prune(fileSystem.getPath("test")), + + of("file-1.json", "test.json", "test.txt"), + of("test", "test/file.txt", "test/data.json", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); + } + + @Test + public void all_files_recursive_with_prune_absolute() { + assertFileHelper(FileFinder.files(testRoot()).prune(testRoot().resolve("test/subdir-1")), + + of("file-1.json", "test.json", "test.txt", "test/file.txt", "test/data.json"), + of("test", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); + } + + @Test(expected = IllegalArgumentException.class) + public void throws_if_prune_path_not_under_base_path() { + FileFinder.files(Paths.get("/some/path")).prune(Paths.get("/other/path")); + } + @Test public void with_file_filter_recursive() { assertFileHelper(FileFinder.files(testRoot()) -- cgit v1.2.3 From be1fb0d49eead759e69ed224c403cd8ecf5ed84e Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Fri, 24 Apr 2020 11:18:18 +0200 Subject: Apply suggestions from code review Co-Authored-By: Jon Bratseth --- .../java/com/yahoo/prelude/querytransform/StemmingSearcher.java | 4 ++-- .../main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java | 8 ++++---- .../main/java/com/yahoo/language/opennlp/OptimaizeDetector.java | 2 +- .../src/main/java/com/yahoo/language/simple/SimpleTokenizer.java | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 9b846d9f0ae..7279e1d6a3d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -184,7 +184,7 @@ public class StemmingSearcher extends Searcher { Substring substring = getOffsets(current); if (segments.size() == 1) { - getLogger().log(LogLevel.DEBUG, "Stem '"+current.stringValue()+"' mode "+index.getStemMode() + getLogger().log(LogLevel.DEBUG, () -> "Stem '"+current.stringValue()+"' mode "+index.getStemMode() +" and language '"+context.language+"' -> '"+segments.get(0)+"'"); TaggableItem w = singleWordSegment(current, segments.get(0), index, substring, context.insidePhrase); setMetaData(current, context.reverseConnectivity, w); @@ -206,7 +206,7 @@ public class StemmingSearcher extends Searcher { composite = chooseComposite(current, ((Item) current).getParent(), indexName); for (StemList segment : segments) { - getLogger().log(LogLevel.DEBUG, "Stem to multiple segments '"+segment+"'"); + getLogger().log(LogLevel.DEBUG, () -> "Stem to multiple segments '"+segment+"'"); TaggableItem w = singleWordSegment(current, segment, index, substring, context.insidePhrase); if (composite instanceof AndSegmentItem) { diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java index 0ebd4e0f638..9a1e6da7629 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -60,7 +60,7 @@ public class OpenNlpTokenizer implements Tokenizer { } private Stemmer getStemmerForLanguage(Language language, StemMode stemMode) { - log.log(Level.FINEST, "getStemmerForLanguage '"+language+"' mode: "+stemMode); + log.log(Level.FINEST, () -> "getStemmerForLanguage '"+language+"' mode: "+stemMode); if (language == null || Language.ENGLISH.equals(language) || StemMode.NONE.equals(stemMode)) { return null; } @@ -124,7 +124,7 @@ public class OpenNlpTokenizer implements Tokenizer { private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, Stemmer stemmer) { - log.log(Level.FINEST, "processToken '"+token+"'"); + log.log(Level.FINEST, () -> "processToken '"+token+"'"); token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) @@ -132,9 +132,9 @@ public class OpenNlpTokenizer implements Tokenizer { if (stemMode != StemMode.NONE) { String oldToken = token; token = doStemming(token, stemmer); - log.log(Level.FINEST, "stem '"+oldToken+"' to '"+token+"'"); + log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+token+"'"); } - log.log(Level.FINEST, "processed token is: "+token); + log.log(Level.FINEST, () -> "processed token is: "+token); return token; } diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java index a42c9f0504e..bf07c91ba44 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java @@ -99,7 +99,7 @@ public class OptimaizeDetector implements Detector { private static Language guessLanguageUsingOptimaize(String input) { Optional result = languageDetector.detect(textObjectFactory.forText(input)); if ( ! result.isPresent()) return Language.UNKNOWN; - log.log(Level.FINE, "guessing language "+result.get()+" from input: "+input); + log.log(Level.FINE, () -> "guessing language "+result.get()+" from input: "+input); return Language.fromLocale(new Locale(result.get().getLanguage())); } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index a8470d86869..aa24e359b53 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -75,9 +75,9 @@ public class SimpleTokenizer implements Tokenizer { if (stemMode != StemMode.NONE) { String oldToken = token; token = stemmer.stem(token); - log.log(Level.FINEST, "stem '"+oldToken+"' to '"+token+"'"); + log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+token+"'"); } - log.log(Level.FINEST, "processed token is: "+token); + log.log(Level.FINEST, () -> "processed token is: "+token); return token; } -- cgit v1.2.3 From b1e84e7c2275cf787826e7621934480c7056f142 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 24 Apr 2020 09:29:53 +0000 Subject: variables in lambdas must be final --- .../java/com/yahoo/language/opennlp/OpenNlpTokenizer.java | 13 ++++++++----- .../java/com/yahoo/language/simple/SimpleTokenizer.java | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java index 9a1e6da7629..d3f6fcf2ee3 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -124,18 +124,21 @@ public class OpenNlpTokenizer implements Tokenizer { private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, Stemmer stemmer) { - log.log(Level.FINEST, () -> "processToken '"+token+"'"); + final String original = token; + log.log(Level.FINEST, () -> "processToken '"+original+"'"); token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); if (stemMode != StemMode.NONE) { - String oldToken = token; + final String oldToken = token; token = doStemming(token, stemmer); - log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+token+"'"); + final String newToken = token; + log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+newToken+"'"); } - log.log(Level.FINEST, () -> "processed token is: "+token); - return token; + final String result = token; + log.log(Level.FINEST, () -> "processed token is: "+result); + return result; } private String doStemming(String token, Stemmer stemmer) { diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index aa24e359b53..7df432f496d 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -67,18 +67,21 @@ public class SimpleTokenizer implements Tokenizer { } private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) { - log.log(Level.FINEST, "processToken '"+token+"'"); + final String original = token; + log.log(Level.FINEST, () -> "processToken '"+original+"'"); token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); if (stemMode != StemMode.NONE) { - String oldToken = token; + final String oldToken = token; token = stemmer.stem(token); - log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+token+"'"); + final String newToken = token; + log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+newToken+"'"); } - log.log(Level.FINEST, () -> "processed token is: "+token); - return token; + final String result = token; + log.log(Level.FINEST, () -> "processed token is: "+result); + return result; } } -- cgit v1.2.3 From 741e3b087ebe87c3288b92800f52458dec20a493 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 23 Apr 2020 18:08:10 +0200 Subject: Cleanup --- .../node/admin/configserver/noderepository/NodeMembership.java | 5 ----- .../yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java | 5 ----- .../yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java | 1 - .../com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java | 4 +++- .../yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java | 2 +- 5 files changed, 4 insertions(+), 13 deletions(-) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java index 3d89b3a991a..93541b03537 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java @@ -25,11 +25,6 @@ public class NodeMembership { return clusterType; } - /** DEPRECATED: Use {@link #type()} instead */ - public String clusterType() { - return clusterType.value(); - } - public String clusterId() { return clusterId; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 9c718955c0e..184858ab747 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -73,11 +73,6 @@ public class StorageMaintainer { this.clock = clock; } - // TODO: Remove, use diskUsageFor() instead - public Optional getDiskUsageFor(NodeAgentContext context) { - return diskUsageFor(context).map(DiskSize::bytes); - } - public Optional diskUsageFor(NodeAgentContext context) { try { DiskSize cachedDiskUsage = diskUsage.getIfPresent(context.containerName()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java index c9a35d09bff..0ce6f1e6b45 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java @@ -20,7 +20,6 @@ import static com.yahoo.yolean.Exceptions.uncheck; public class DiskCleanup { private static final Logger logger = Logger.getLogger(DiskCleanup.class.getName()); - private static final char[] UNITS = "kMGTPE".toCharArray(); private static final Comparator PRIORITIZED_FILE_ATTRIBUTES_COMPARATOR = Comparator .comparing(PrioritizedFileAttributes::priority) .thenComparingLong(f -> f.fileAttributes().size()) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java index e6d8835ef9c..40a11f61d3f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java @@ -1,5 +1,7 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.file; +import java.util.Locale; import java.util.Objects; /** @@ -40,7 +42,7 @@ public class DiskSize { int unit = -1; double remaining = bytes; for (; remaining >= 1000; unit++) remaining /= 1000; - return String.format("%." + decimals + "f %sB", remaining, UNITS[unit]); + return String.format(Locale.ENGLISH, "%." + decimals + "f %sB", remaining, UNITS[unit]); } @Override diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 9b509829ba2..a1ea3558504 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -66,7 +66,7 @@ public class DockerTester implements AutoCloseable { DockerTester() { - when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.empty()); + when(storageMaintainer.diskUsageFor(any())).thenReturn(Optional.empty()); IPAddressesMock ipAddresses = new IPAddressesMock(); ipAddresses.addAddress(HOST_HOSTNAME.value(), "1.1.1.1"); -- cgit v1.2.3 From 4a4b2a8e0fe79114d9e9508df2ebdd663c44e32c Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 24 Apr 2020 11:39:19 +0200 Subject: Only print docker image if wanted is not the same as current --- .../yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 353b82ede49..bc206011bb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -554,7 +554,9 @@ public class InternalStepRunner implements StepRunner { } private String currentPlatform(Node node) { - return node.wantedDockerImage().repository() + ":" + node.wantedVersion(); + String currentRepo = node.currentDockerImage().repository(); + String wantedRepo = node.wantedDockerImage().repository(); + return (currentRepo.equals(wantedRepo) ? "" : currentRepo + ":") + node.currentVersion(); } private String humanize(Node.ServiceState state) { -- cgit v1.2.3 From 2012293d01c1749b6e202666358669a64c1958e4 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 24 Apr 2020 11:56:01 +0200 Subject: Add application resource to root --- .../com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java | 2 +- .../com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index e275d978e4e..94e52c758eb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -107,7 +107,7 @@ public class NodesApiHandler extends LoggingRequestHandler { private HttpResponse handleGET(HttpRequest request) { Path path = new Path(request.getUri()); String pathS = request.getUri().getPath(); - if (pathS.equals( "/nodes/v2/")) return new ResourceResponse(request.getUri(), "state", "node", "command", "maintenance", "upgrade"); + if (pathS.equals( "/nodes/v2/")) return new ResourceResponse(request.getUri(), "state", "node", "command", "maintenance", "upgrade", "application"); if (pathS.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository); if (pathS.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository); if (pathS.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, orchestrator, nodeRepository); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json index 86becefb146..cc91314df84 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json @@ -14,6 +14,9 @@ }, { "url":"http://localhost:8080/nodes/v2/upgrade/" + }, + { + "url":"http://localhost:8080/nodes/v2/application/" } ] } \ No newline at end of file -- cgit v1.2.3 From bba1055e9326e70a54065bb07af6b16fcba69b5c Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 24 Apr 2020 12:06:31 +0200 Subject: Test applications response --- .../hosted/provision/restapi/v2/NodesApiHandler.java | 4 ++-- .../hosted/provision/restapi/v2/RestApiTest.java | 8 +++++++- .../provision/restapi/v2/responses/applications.json | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/applications.json diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 94e52c758eb..1a73d1b77e7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -408,7 +408,7 @@ public class NodesApiHandler extends LoggingRequestHandler { Cursor applications = root.setArray("applications"); for (ApplicationId id : nodeRepository.applications().ids()) { Cursor application = applications.addObject(); - application.setString("url", withPath("nodes/v2/applications/" + id.toFullString(), uri).toString()); + application.setString("url", withPath("/nodes/v2/applications/" + id.toFullString(), uri).toString()); application.setString("id", id.toFullString()); } return new SlimeJsonResponse(slime); @@ -419,7 +419,7 @@ public class NodesApiHandler extends LoggingRequestHandler { if (application.isEmpty()) return ErrorResponse.notFoundError("No application '" + id + "'"); Slime slime = ApplicationSerializer.toSlime(application.get(), - withPath("nodes/v2/applications/" + id, uri)); + withPath("/nodes/v2/applications/" + id, uri)); return new SlimeJsonResponse(slime); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 0001c344dd3..01719450e2b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -56,7 +56,7 @@ public class RestApiTest { if (container != null) container.close(); } - /** This test gives examples of all the requests that can be made to nodes/v2 */ + /** This test gives examples of the node requests that can be made to nodes/v2 */ @Test public void test_requests() throws Exception { // GET @@ -231,6 +231,12 @@ public class RestApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); } + @Test + public void test_application_requests() throws Exception { + + assertFile(new Request("http://localhost:8080/nodes/v2/application/"), "applications.json"); + } + @Test public void maintenance_requests() throws Exception { // POST deactivation of a maintenance job diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/applications.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/applications.json new file mode 100644 index 00000000000..6d25b3e59f5 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/applications.json @@ -0,0 +1,20 @@ +{ + "applications" : [ + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant4.application4.instance4", + "id" : "tenant4.application4.instance4" + }, + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant2.application2.instance2", + "id" : "tenant2.application2.instance2" + }, + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant3.application3.instance3", + "id" : "tenant3.application3.instance3" + }, + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant1.application1.instance1", + "id" : "tenant1.application1.instance1" + } + ] +} \ No newline at end of file -- cgit v1.2.3 From d8ec2c5c7dc4836dc76226bdddb71b3914fd4bad Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 24 Apr 2020 12:51:34 +0200 Subject: Test application responses --- .../com/yahoo/config/provision/ApplicationId.java | 2 +- .../provision/restapi/v2/NodesApiHandler.java | 2 +- .../provision/testutils/MockNodeRepository.java | 22 +++++++-- .../vespa/hosted/provision/testutils/README.md | 2 +- .../hosted/provision/restapi/v2/RestApiTest.java | 5 +- .../restapi/v2/responses/application1.json | 56 ++++++++++++++++++++++ .../restapi/v2/responses/application2.json | 32 +++++++++++++ 7 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application1.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2.json diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index ef89c1d7525..692c941877b 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -48,7 +48,7 @@ public final class ApplicationId implements Comparable { } public static ApplicationId fromFullString(String idString) { - String[] parts = idString.split("."); + String[] parts = idString.split("\\."); if (parts.length < 3) throw new IllegalArgumentException("Application ids must be on the form tenant.application.instance, but was " + idString); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 1a73d1b77e7..6799b68daae 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -119,7 +119,7 @@ public class NodesApiHandler extends LoggingRequestHandler { if (pathS.startsWith("/nodes/v2/capacity")) return new HostCapacityResponse(nodeRepository, request); if (path.matches("/nodes/v2/application")) return applicationList(request.getUri()); if (path.matches("/nodes/v2/application/{applicationId}")) return application(path.get("applicationId"), request.getUri()); - throw new NotFoundException("Nothing at path '" + path + "'"); + throw new NotFoundException("Nothing at " + path); } private HttpResponse handlePUT(HttpRequest request) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 8ff17a8e5a3..0cafe35ee52 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -16,11 +16,14 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Status; @@ -142,9 +145,22 @@ public class MockNodeRepository extends NodeRepository { ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build(); activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), null), zoneApp, provisioner); - ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); - ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")).vespaVersion("6.42").build(); - provisioner.prepare(app1, cluster1, Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1))), null); + ApplicationId app1Id = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); + ClusterSpec cluster1Id = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")).vespaVersion("6.42").build(); + provisioner.prepare(app1Id, + cluster1Id, + Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1)), + new ClusterResources(8, 2, new NodeResources(4, 16, 1000, 1)), false, true), + null); + Application app1 = applications().get(app1Id).get(); + Cluster cluster1 = app1.cluster(cluster1Id.id()).get(); + cluster1 = cluster1.withSuggested(new ClusterResources(6, 2, + new NodeResources(3, 20, 100, 1))); + cluster1 = cluster1.withTarget(new ClusterResources(4, 1, + new NodeResources(3, 16, 100, 1))); + try (Mutex lock = lock(app1Id)) { + applications().put(app1.with(cluster1), lock); + } ApplicationId app2 = ApplicationId.from(TenantName.from("tenant2"), ApplicationName.from("application2"), InstanceName.from("instance2")); ClusterSpec cluster2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id2")).vespaVersion("6.42").build(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md index bbb2ec7080b..f7421bf07c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md @@ -1,5 +1,5 @@ -The test resources are used by both NoadAdmin and NodeRepository +The test resources are used by both NodeAdmin and NodeRepository tests to verify APIs. So when modifying this test data remember to check tests for both NodeAdmin and NodeRepository. diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 01719450e2b..838a39ba01e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -233,8 +233,11 @@ public class RestApiTest { @Test public void test_application_requests() throws Exception { - assertFile(new Request("http://localhost:8080/nodes/v2/application/"), "applications.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/application/tenant1.application1.instance1"), + "application1.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/application/tenant2.application2.instance2"), + "application2.json"); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application1.json new file mode 100644 index 00000000000..ce2309e4d6c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application1.json @@ -0,0 +1,56 @@ +{ + "url" : "http://localhost:8080/nodes/v2/applications/tenant1.application1.instance1", + "id" : "tenant1.application1.instance1", + "clusters" : { + "id1" : { + "min" : { + "nodes" : 2, + "groups" : 1, + "resources" : { + "vcpu" : 2.0, + "memoryGb" : 8.0, + "diskGb" : 50.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + }, + "max" : { + "nodes" : 8, + "groups" : 2, + "resources" : { + "vcpu" : 4.0, + "memoryGb" : 16.0, + "diskGb" : 1000.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + }, + "suggested" : { + "nodes" : 6, + "groups" : 2, + "resources" : { + "vcpu" : 3.0, + "memoryGb" : 20.0, + "diskGb" : 100.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + }, + "target" : { + "nodes" : 4, + "groups" : 1, + "resources" : { + "vcpu" : 3.0, + "memoryGb" : 16.0, + "diskGb" : 100.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + } + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2.json new file mode 100644 index 00000000000..75af5d4a328 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2.json @@ -0,0 +1,32 @@ +{ + "url": "http://localhost:8080/nodes/v2/applications/tenant2.application2.instance2", + "id": "tenant2.application2.instance2", + "clusters": { + "id2": { + "min": { + "nodes": 2, + "groups": 1, + "resources": { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "fast", + "storageType": "any" + } + }, + "max": { + "nodes": 2, + "groups": 1, + "resources": { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "fast", + "storageType": "any" + } + } + } + } +} -- cgit v1.2.3 From 27a1f926c60f7684d5ec17988dc17681b0c73d30 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Fri, 24 Apr 2020 11:13:48 +0000 Subject: Optimize B-tree bucket DB lookup with used-bits aggregation By tracking the minimum used bits count across all buckets in the database we can immediately start seeking at that implicit level in the tree, as we know no parent buckets can exist above that level. Local synthetic benchmarking shows the following results with a DB size of 917504 buckets and performing getParents for all buckets in sequence: Before optimization: - B-tree DB: 0.593321 seconds - Legacy DB: 0.227947 seconds After optimization: - B-tree DB: 0.213738 seconds - Legacy DB: (unchanged) --- .../src/tests/distributor/bucketdatabasetest.cpp | 33 +++++++++++++++++ .../storage/bucketdb/btree_bucket_database.cpp | 41 ++++++++++++++++------ .../vespa/storage/bucketdb/btree_bucket_database.h | 17 ++++++++- 3 files changed, 79 insertions(+), 12 deletions(-) diff --git a/storage/src/tests/distributor/bucketdatabasetest.cpp b/storage/src/tests/distributor/bucketdatabasetest.cpp index 1d5355df92f..56e996abf99 100644 --- a/storage/src/tests/distributor/bucketdatabasetest.cpp +++ b/storage/src/tests/distributor/bucketdatabasetest.cpp @@ -707,4 +707,37 @@ TEST_P(BucketDatabaseTest, DISABLED_benchmark_const_iteration) { db().toString(false).c_str(), elapsed); } +TEST_P(BucketDatabaseTest, DISABLED_benchmark_find_parents) { + constexpr uint32_t superbuckets = 1u << 16u; + constexpr uint32_t sub_buckets = 14; + constexpr uint32_t n_buckets = superbuckets * sub_buckets; + + std::vector bucket_keys; + bucket_keys.reserve(n_buckets); + + for (uint32_t sb = 0; sb < superbuckets; ++sb) { + for (uint64_t i = 0; i < sub_buckets; ++i) { + document::BucketId bucket(48, (i << 32ULL) | sb); // TODO eval with different bit counts + bucket_keys.emplace_back(bucket.toKey()); + } + } + fprintf(stderr, "Inserting %zu buckets into DB\n", bucket_keys.size()); + std::sort(bucket_keys.begin(), bucket_keys.end()); + for (uint64_t k : bucket_keys) { + db().update(BucketDatabase::Entry(BucketId(BucketId::keyToBucketId(k)), BI3(0, 1, 2))); + } + + fprintf(stderr, "Invoking getParents() %zu times\n", bucket_keys.size()); + auto elapsed = vespalib::BenchmarkTimer::benchmark([&] { + std::vector entries; + for (uint64_t k : bucket_keys) { + db().getParents(BucketId(BucketId::keyToBucketId(k)), entries); + assert(entries.size() == 1); + entries.clear(); + } + }, 30); + fprintf(stderr, "Looking up all buckets in %s takes %g seconds\n", + db().toString(false).c_str(), elapsed); +} + } diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp index 85f5c8c5be9..2adb3049853 100644 --- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp +++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp @@ -194,13 +194,27 @@ void BTreeBucketDatabase::remove(const BucketId& bucket) { * This allows us to reuse a single iterator and to continue seeking forwards * from its current position. * + * To speed up the process of converging on the target bucket without needing + * to check many unrelated subtrees, we let the underlying B-tree automatically + * aggregate the min/max range of the used-bits of all contained bucket keys. + * If we e.g. know that the minimum number of used bits in the DB is 16, we can + * immediately seek to this level in the tree instead of working our way down + * one bit at a time. By definition, no parents can exist above this level. + * This is a very important optimization, as bucket trees are usually very well + * balanced due to randomized distribution of data (combined with a cluster-wide + * minimum tree level imposed by distribution bits). It is common that the minimum + * number of used bits == max number of used bits, i.e. a totally even split. + * This means that for a system without inconsistently split buckets (i.e. no + * parents) we're highly likely to converge on the target bucket in a single seek. + * * Algorithm: * * Core invariant: every subsequent iterator seek performed in this algorithm * is for a key that is strictly higher than the one the iterator is currently at. * * 1. Lbound seek to the lowest key that is known to exclude all already visited - * parents. On the first iteration this is zero, i.e. the first in-order node. + * parents. On the first iteration we use a bit count equal to the minimum number + * of key used-bits in the entire DB, allowing us to potentially skip most subtrees. * 2. If the current node's key is greater than that of the requested bucket's key, * we've either descended to--or beyond--it in its own subtree or we've entered * a disjoint subtree. Since we know that all parents must sort before any given @@ -229,14 +243,13 @@ void BTreeBucketDatabase::remove(const BucketId& bucket) { * 6. Iff iterator is still valid, go to step 2 * * This algorithm is able to skip through large parts of the tree in a sparsely populated - * tree, but the number of seeks will trend towards O(b) as with the legacy implementation - * when a tree is densely populated. This because all logical inner nodes in the tree will - * have subtrees under them. Even in the worst case we should be more efficient than the - * legacy implementation since we've cut any dense search space in half for each invocation - * of seek() on the iterator - * - * TODO use min-aggregation to infer used bit range in the tree. This should allow us to - * skip scanning through many disjoint subtrees. + * tree, but the number of seeks will trend towards O(b - min_bits) as with the legacy + * implementation when a tree is densely populated (where `b` is the used-bits count of the + * most specific node in the tree for the target bucket, and min_bits is the minimum number + * of used-bits for any key in the database). This because all logical inner nodes in the tree + * will have subtrees under them. Even in the worst case we should be more efficient than the + * legacy Judy-based implementation since we've cut any dense search space in half for each + * invocation of seek() on the iterator. */ BTreeBucketDatabase::BTree::ConstIterator BTreeBucketDatabase::find_parents_internal(const BTree::FrozenView& frozen_view, @@ -245,9 +258,15 @@ BTreeBucketDatabase::find_parents_internal(const BTree::FrozenView& frozen_view, { const uint64_t bucket_key = bucket.toKey(); auto iter = frozen_view.begin(); - // Start at the root level, descending towards the bucket itself. + if (!iter.valid()) { + return iter; + } + assert(iter.getAggregated().getMin() >= static_cast(BucketId::minNumBits)); + assert(iter.getAggregated().getMax() <= static_cast(BucketId::maxNumBits)); + // Start at the lowest possible tree level no parents can exist above, + // descending towards the bucket itself. // Try skipping as many levels of the tree as possible as we go. - uint32_t bits = 1; + uint32_t bits = iter.getAggregated().getMin(); while (iter.valid() && (iter.getKey() < bucket_key)) { auto candidate = BucketId(BucketId::keyToBucketId(iter.getKey())); if (candidate.contains(bucket)) { diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.h b/storage/src/vespa/storage/bucketdb/btree_bucket_database.h index 8898b0c395a..75736e59092 100644 --- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.h +++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.h @@ -4,6 +4,8 @@ #include "bucketdatabase.h" #include +#include +#include #include namespace storage { @@ -23,8 +25,21 @@ namespace storage { */ // TODO create and use a new DB interface with better bulk loading, snapshot and iteration support class BTreeBucketDatabase : public BucketDatabase { + + struct KeyUsedBitsMinMaxAggrCalc : search::btree::MinMaxAggrCalc { + constexpr static bool aggregate_over_values() { return false; } + constexpr static int32_t getVal(uint64_t key) noexcept { + static_assert(document::BucketId::CountBits == 6u); + return static_cast(key & 0b11'1111U); // 6 LSB of key contains used-bits + } + }; + // Mapping from u64: bucket key -> - using BTree = search::btree::BTree; + using BTree = search::btree::BTree, + search::btree::BTreeDefaultTraits, + KeyUsedBitsMinMaxAggrCalc>; using ReplicaStore = search::datastore::ArrayStore; using GenerationHandler = vespalib::GenerationHandler; -- cgit v1.2.3 From 47e3cfe6c39985ba2db5204248fc46652b76bb67 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 24 Apr 2020 13:42:41 +0200 Subject: Remove unused metrics --- .../controller/maintenance/MetricsReporter.java | 23 +----- .../maintenance/MetricsReporterTest.java | 94 ++++++++++------------ 2 files changed, 44 insertions(+), 73 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index d70c65f343a..dfd35d52b2d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -40,22 +40,11 @@ public class MetricsReporter extends Maintainer { public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades"; public static final String DEPLOYMENT_BUILD_AGE_SECONDS = "deployment.buildAgeSeconds"; public static final String DEPLOYMENT_WARNINGS = "deployment.warnings"; - // TODO(mpolden): Remove these two metrics - public static final String NODES_FAILING_PLATFORM_UPGRADE = "deployment.nodesFailingSystemUpgrade"; - public static final String NODES_FAILING_OS_UPGRADE = "deployment.nodesFailingOsUpgrade"; public static final String OS_CHANGE_DURATION = "deployment.osChangeDuration"; public static final String PLATFORM_CHANGE_DURATION = "deployment.platformChangeDuration"; public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; - // The time a system application node can spend after suspending for Vespa upgrade until the upgrade is completed. - // Nodes exceeding this budget are counted as failures. - private static final Duration PLATFORM_UPGRADE_BUDGET = Duration.ofMinutes(30); - - // The time a system application node can spend after suspending foor OS upgrade until the upgrade is completed. - // Nodes exceeding this budget are counted as failures. - private static final Duration OS_UPGRADE_BUDGET = Duration.ofMinutes(30); - private final Metric metric; private final Clock clock; @@ -113,16 +102,8 @@ public class MetricsReporter extends Maintainer { } private void reportChangeDurations() { - var platformChangeDurations = platformChangeDurations(); - var osChangeDurations = osChangeDurations(); - var nodesFailingSystemUpgrade = platformChangeDurations.values().stream() - .filter(duration -> duration.compareTo(PLATFORM_UPGRADE_BUDGET) > 0) - .count(); - var nodesFailingOsUpgrade = osChangeDurations.values().stream() - .filter(duration -> duration.compareTo(OS_UPGRADE_BUDGET) > 0) - .count(); - metric.set(NODES_FAILING_PLATFORM_UPGRADE, nodesFailingSystemUpgrade, metric.createContext(Map.of())); - metric.set(NODES_FAILING_OS_UPGRADE, nodesFailingOsUpgrade, metric.createContext(Map.of())); + Map platformChangeDurations = platformChangeDurations(); + Map osChangeDurations = osChangeDurations(); platformChangeDurations.forEach((nodeVersion, duration) -> { metric.set(PLATFORM_CHANGE_DURATION, duration.toSeconds(), metric.createContext(dimensions(nodeVersion))); }); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index e2482fc9174..df5c3bcbc3c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -240,26 +240,27 @@ public class MetricsReporterTest { var version0 = Version.fromString("7.0"); tester.upgradeSystem(version0); reporter.maintain(); - assertEquals(0, getNodesFailingUpgrade()); + var hosts = tester.configServer().nodeRepository().list(zone, SystemApplication.configServer.id()); + assertPlatformChangeDuration(Duration.ZERO, hosts); for (var version : List.of(Version.fromString("7.1"), Version.fromString("7.2"))) { // System starts upgrading to next version tester.upgradeController(version); reporter.maintain(); - assertEquals(0, getNodesFailingUpgrade()); + assertPlatformChangeDuration(Duration.ZERO, hosts); systemUpgrader.maintain(); // 30 minutes pass and nothing happens tester.clock().advance(Duration.ofMinutes(30)); runAll(tester::computeVersionStatus, reporter); - assertEquals(0, getNodesFailingUpgrade()); + assertPlatformChangeDuration(Duration.ZERO, hosts); // 1/3 nodes upgrade within timeout - var hosts = tester.configServer().nodeRepository().list(zone, SystemApplication.configServer.id()); - assertEquals("Wanted version is raised for all nodes", version, hosts.stream() - .map(Node::wantedVersion) - .min(Comparator.naturalOrder()) - .get()); + assertEquals("Wanted version is raised for all nodes", version, + getNodes(zone, hosts, tester).stream() + .map(Node::wantedVersion) + .min(Comparator.naturalOrder()) + .get()); suspend(hosts, zone, tester); var firstHost = hosts.get(0); upgradeTo(version, List.of(firstHost), zone, tester); @@ -267,21 +268,13 @@ public class MetricsReporterTest { // 2/3 spend their budget and are reported as failures tester.clock().advance(Duration.ofHours(1)); runAll(tester::computeVersionStatus, reporter); - assertEquals(2, getNodesFailingUpgrade()); - assertEquals("Change duration is reset for completed upgrade", Duration.ZERO, - getChangeDuration(MetricsReporter.PLATFORM_CHANGE_DURATION, firstHost.hostname())); - for (var host : hosts.subList(1, hosts.size())) { - assertEquals(Duration.ofHours(1), getChangeDuration(MetricsReporter.PLATFORM_CHANGE_DURATION, - host.hostname())); - } + assertPlatformChangeDuration(Duration.ZERO, List.of(firstHost)); + assertPlatformChangeDuration(Duration.ofHours(1), hosts.subList(1, hosts.size())); // Remaining nodes eventually upgrade - upgradeTo(version, hosts.subList(1, 3), zone, tester); + upgradeTo(version, hosts.subList(1, hosts.size()), zone, tester); runAll(tester::computeVersionStatus, reporter); - assertEquals(0, getNodesFailingUpgrade()); - for (var host : hosts) { - assertEquals(Duration.ZERO, getChangeDuration(MetricsReporter.PLATFORM_CHANGE_DURATION, host.hostname())); - } + assertPlatformChangeDuration(Duration.ZERO, hosts); assertEquals(version, tester.controller().systemVersion()); } } @@ -306,18 +299,19 @@ public class MetricsReporterTest { tester.configServer().setOsVersion(version0, SystemApplication.tenantHost.id(), zone); tester.configServer().setOsVersion(version0, SystemApplication.configServerHost.id(), zone); runAll(statusUpdater, reporter); - assertEquals(0, getNodesFailingOsUpgrade()); + List hosts = tester.configServer().nodeRepository().list(zone); + assertOsChangeDuration(Duration.ZERO, hosts); for (var version : List.of(Version.fromString("8.1"), Version.fromString("8.2"))) { // System starts upgrading to next OS version tester.controller().upgradeOsIn(cloud, version, false); runAll(osUpgrader, statusUpdater, reporter); - assertEquals(0, getNodesFailingOsUpgrade()); + assertOsChangeDuration(Duration.ZERO, hosts); // Over 30 minutes pass and nothing happens tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); runAll(statusUpdater, reporter); - assertEquals(0, getNodesFailingOsUpgrade()); + assertOsChangeDuration(Duration.ZERO, hosts); // Nodes are told to upgrade, but do not suspend yet assertEquals("Wanted OS version is raised for all nodes", version, @@ -330,52 +324,40 @@ public class MetricsReporterTest { // Another 30 minutes pass tester.clock().advance(Duration.ofMinutes(30)); runAll(statusUpdater, reporter); - assertEquals(0, getNodesFailingOsUpgrade()); + assertOsChangeDuration(Duration.ZERO, hosts); // 3/6 hosts suspend - var hosts = tester.configServer().nodeRepository().list(zone); var suspendedHosts = hosts.subList(0, 3); suspend(suspendedHosts, zone, tester); runAll(statusUpdater, reporter); - assertEquals(0, getNodesFailingOsUpgrade()); + assertOsChangeDuration(Duration.ZERO, hosts); - // Two hosts upgrade after 20 minutes + // Two hosts spend 20 minutes upgrading var hostsUpgraded = suspendedHosts.subList(0, 2); tester.clock().advance(Duration.ofMinutes(20)); runAll(statusUpdater, reporter); - for (var host : suspendedHosts) { - assertEquals(Duration.ofMinutes(20), getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, host.hostname())); - } + assertOsChangeDuration(Duration.ofMinutes(20), hostsUpgraded); upgradeOsTo(version, hostsUpgraded, zone, tester); runAll(statusUpdater, reporter); - assertEquals(0, getNodesFailingOsUpgrade()); - for (var host : hostsUpgraded) { - assertEquals(Duration.ZERO, getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, host.hostname())); - } + assertOsChangeDuration(Duration.ZERO, hostsUpgraded); // One host consumes budget without upgrading var brokenHost = suspendedHosts.get(2); tester.clock().advance(Duration.ofMinutes(15)); runAll(statusUpdater, reporter); - assertEquals(1, getNodesFailingOsUpgrade()); - assertEquals(Duration.ofMinutes(35), getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, - brokenHost.hostname())); + assertOsChangeDuration(Duration.ofMinutes(35), List.of(brokenHost)); // Host eventually upgrades and is no longer reported upgradeOsTo(version, List.of(brokenHost), zone, tester); runAll(statusUpdater, reporter); - assertEquals(0, getNodesFailingOsUpgrade()); - assertEquals(Duration.ZERO, getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, brokenHost.hostname())); + assertOsChangeDuration(Duration.ZERO, List.of(brokenHost)); // Remaining hosts suspend and upgrade successfully var remainingHosts = hosts.subList(3, hosts.size()); suspend(remainingHosts, zone, tester); upgradeOsTo(version, remainingHosts, zone, tester); runAll(statusUpdater, reporter); - assertEquals(0, getNodesFailingOsUpgrade()); - for (var host : hosts) { - assertEquals(Duration.ZERO, getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, host.hostname())); - } + assertOsChangeDuration(Duration.ZERO, hosts); } } @@ -404,11 +386,15 @@ public class MetricsReporterTest { zone, tester); } + private List getNodes(ZoneId zone, List nodes, ControllerTester tester) { + return tester.configServer().nodeRepository().list(zone, nodes.stream() + .map(Node::hostname) + .collect(Collectors.toList())); + } + private void updateNodes(List nodes, UnaryOperator builderOps, ZoneId zone, ControllerTester tester) { - var currentNodes = tester.configServer().nodeRepository().list(zone, nodes.stream() - .map(Node::hostname) - .collect(Collectors.toList())); + var currentNodes = getNodes(zone, nodes, tester); var updatedNodes = currentNodes.stream() .map(node -> builderOps.apply(new Node.Builder(node)).build()) .collect(Collectors.toList()); @@ -427,18 +413,22 @@ public class MetricsReporterTest { return getMetric(MetricsReporter.DEPLOYMENT_WARNINGS, id).intValue(); } - private int getNodesFailingUpgrade() { - return metrics.getMetric(MetricsReporter.NODES_FAILING_PLATFORM_UPGRADE).intValue(); - } - private Duration getChangeDuration(String metric, HostName hostname) { return metrics.getMetric((dimensions) -> hostname.value().equals(dimensions.get("host")), metric) .map(n -> Duration.ofSeconds(n.longValue())) .orElseThrow(() -> new IllegalArgumentException("Expected to find metric for " + hostname)); } - private int getNodesFailingOsUpgrade() { - return metrics.getMetric(MetricsReporter.NODES_FAILING_OS_UPGRADE).intValue(); + private void assertPlatformChangeDuration(Duration duration, List nodes) { + for (var node : nodes) { + assertEquals(duration, getChangeDuration(MetricsReporter.PLATFORM_CHANGE_DURATION, node.hostname())); + } + } + + private void assertOsChangeDuration(Duration duration, List nodes) { + for (var node : nodes) { + assertEquals(duration, getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, node.hostname())); + } } private Number getMetric(String name, ApplicationId id) { -- cgit v1.2.3 From fd31eb82f85582bc7563a9fc03b77e2d03879c9c Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 24 Apr 2020 14:15:04 +0200 Subject: Cleer outdated suggestions --- .../maintenance/ScalingSuggestionsMaintainer.java | 31 +++++++++++++++------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index f80f2d11753..985c5c7d1a7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -50,23 +50,34 @@ public class ScalingSuggestionsMaintainer extends Maintainer { suggest(application, clusterId, clusterNodes)); } + private Applications applications() { + return nodeRepository().applications(); + } + private void suggest(ApplicationId applicationId, ClusterSpec.Id clusterId, List clusterNodes) { - Applications applications = nodeRepository().applications(); - Application application = applications.get(applicationId).orElse(new Application(applicationId)); - Cluster cluster = application.clusters().get(clusterId); - if (cluster == null) return; - Optional target = autoscaler.suggest(cluster, clusterNodes); - if (target.isEmpty()) return; - ClusterResources suggestion = target.get().toAdvertisedClusterResources(); - + Application application = applications().get(applicationId).orElse(new Application(applicationId)); + Optional cluster = application.cluster(clusterId); + if (cluster.isEmpty()) return; + Optional suggestion = autoscaler.suggest(cluster.get(), clusterNodes); try (Mutex lock = nodeRepository().lock(applicationId)) { - applications.get(applicationId).ifPresent(a -> a.cluster(clusterId).ifPresent(c -> - applications.put(a.with(c.withSuggested(suggestion)), lock))); + applications().get(applicationId).ifPresent(a -> storeSuggestion(suggestion, clusterId, a, lock)); } } + private void storeSuggestion(Optional suggestion, + ClusterSpec.Id clusterId, + Application application, + Mutex lock) { + Optional cluster = application.cluster(clusterId); + if (cluster.isEmpty()) return; + if (suggestion.isPresent()) + applications().put(application.with(cluster.get().withSuggested(suggestion.get().toAdvertisedClusterResources())), lock); + else + applications().put(application.with(cluster.get().withoutSuggested()), lock); + } + private Map> nodesByCluster(List applicationNodes) { return applicationNodes.stream().collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster().id())); } -- cgit v1.2.3 From c09675a41768642d37390c00cb05ecfc533ca5d1 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 24 Apr 2020 14:31:22 +0200 Subject: Ignore hnsw index attribute save file if major parameters are changed. --- .../src/vespa/searchcommon/attribute/config.h | 4 + .../tensorattribute/tensorattribute_test.cpp | 102 ++++++++++++++++++--- .../searchlib/tensor/dense_tensor_attribute.cpp | 20 +++- 3 files changed, 109 insertions(+), 17 deletions(-) diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h index 836fcfed84a..df5aa9e217a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchcommon/src/vespa/searchcommon/attribute/config.h @@ -71,6 +71,10 @@ public: _hnsw_index_params = params; return *this; } + Config& clear_hnsw_index_params() { + _hnsw_index_params.reset(); + return *this; + } /** * Enable attribute posting list to consist of a bitvector in diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 39a7e53ca8c..a008bc73e93 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -215,6 +215,7 @@ struct Fixture { Config _cfg; vespalib::string _name; vespalib::string _typeSpec; + bool _use_mock_index; std::unique_ptr _index_factory; std::shared_ptr _tensorAttr; std::shared_ptr _attr; @@ -229,27 +230,45 @@ struct Fixture { _cfg(BasicType::TENSOR, CollectionType::SINGLE), _name(attr_name), _typeSpec(typeSpec), - _index_factory(std::make_unique()), + _use_mock_index(use_mock_index), + _index_factory(), _tensorAttr(), _attr(), _denseTensors(false), _useDenseTensorAttribute(useDenseTensorAttribute) { - _cfg.setTensorType(ValueType::from_spec(typeSpec)); + if (enable_hnsw_index) { + _cfg.set_hnsw_index_params(HnswIndexParams(4, 20, DistanceMetric::Euclidean)); + } + setup(); + } + + ~Fixture() {} + + void setup() { + _cfg.setTensorType(ValueType::from_spec(_typeSpec)); if (_cfg.tensorType().is_dense()) { _denseTensors = true; } - if (enable_hnsw_index) { - _cfg.set_hnsw_index_params(HnswIndexParams(4, 20, DistanceMetric::Euclidean)); - if (use_mock_index) { - _index_factory = std::make_unique(); - } + if (_use_mock_index) { + _index_factory = std::make_unique(); + } else { + _index_factory = std::make_unique(); } _tensorAttr = makeAttr(); _attr = _tensorAttr; _attr->addReservedDoc(); } - ~Fixture() {} + + void set_hnsw_index_params(const HnswIndexParams ¶ms) { + _cfg.set_hnsw_index_params(params); + setup(); + } + + void disable_hnsw_index() { + _cfg.clear_hnsw_index_params(); + setup(); + } std::shared_ptr makeAttr() { if (_useDenseTensorAttribute) { @@ -364,6 +383,9 @@ struct Fixture { return denseSpec; } + void set_example_tensors(); + void assert_example_tensors(); + void save_example_tensors_with_mock_index(); void testEmptyAttribute(); void testSetTensorValue(); void testSaveLoad(); @@ -372,6 +394,30 @@ struct Fixture { void testEmptyTensor(); }; + +void +Fixture::set_example_tensors() +{ + set_tensor(1, vec_2d(3, 5)); + set_tensor(2, vec_2d(7, 9)); +} + +void +Fixture::assert_example_tensors() +{ + assertGetTensor(vec_2d(3, 5), 1); + assertGetTensor(vec_2d(7, 9), 2); +} + +void +Fixture::save_example_tensors_with_mock_index() +{ + set_example_tensors(); + mock_index().save_index_with_value(123); + save(); + EXPECT_TRUE(vespalib::fileExists(_name + ".nnidx")); +} + void Fixture::testEmptyAttribute() { @@ -664,11 +710,7 @@ TEST_F("Memory usage is extracted from index when updating stats on attribute", TEST_F("Nearest neighbor index can be saved to disk and then loaded from file", DenseTensorAttributeMockIndex) { - f.set_tensor(1, vec_2d(3, 5)); - f.set_tensor(2, vec_2d(7, 9)); - f.mock_index().save_index_with_value(123); - f.save(); - EXPECT_TRUE(vespalib::fileExists(attr_name + ".nnidx")); + f.save_example_tensors_with_mock_index(); f.load(); // index is loaded from saved file auto& index = f.mock_index(); @@ -678,8 +720,7 @@ TEST_F("Nearest neighbor index can be saved to disk and then loaded from file", TEST_F("onLoad() reconstructs nearest neighbor index if save file does not exists", DenseTensorAttributeMockIndex) { - f.set_tensor(1, vec_2d(3, 5)); - f.set_tensor(2, vec_2d(7, 9)); + f.set_example_tensors(); f.save(); EXPECT_FALSE(vespalib::fileExists(attr_name + ".nnidx")); @@ -689,5 +730,36 @@ TEST_F("onLoad() reconstructs nearest neighbor index if save file does not exist index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); } +TEST_F("onLoads() ignores saved nearest neighbor index if not enabled in config", DenseTensorAttributeMockIndex) +{ + f.save_example_tensors_with_mock_index(); + f.disable_hnsw_index(); + f.load(); + f.assert_example_tensors(); + EXPECT_EQUAL(f.as_dense_tensor().nearest_neighbor_index(), nullptr); +} + +TEST_F("onLoad() ignores saved nearest neightbor index if major index parameters are changed", DenseTensorAttributeMockIndex) +{ + f.save_example_tensors_with_mock_index(); + f.set_hnsw_index_params(HnswIndexParams(5, 20, DistanceMetric::Euclidean)); + f.load(); + f.assert_example_tensors(); + auto& index = f.mock_index(); + EXPECT_EQUAL(0, index.get_index_value()); + index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); +} + +TEST_F("onLoad() uses saved nearest neightbor index if only minor index parameters are changed", DenseTensorAttributeMockIndex) +{ + f.save_example_tensors_with_mock_index(); + f.set_hnsw_index_params(HnswIndexParams(4, 21, DistanceMetric::Euclidean)); + f.load(); + f.assert_example_tensors(); + auto& index = f.mock_index(); + EXPECT_EQUAL(123, index.get_index_value()); + index.expect_adds({}); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 68ce0c1bb00..c9ed4039655 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -59,6 +59,21 @@ TensorReader::is_present() { return true; } +bool +can_use_index_save_file(const search::attribute::Config &config, const search::attribute::AttributeHeader &header) +{ + if (!config.hnsw_index_params().has_value() || !header.get_hnsw_index_params().has_value()) { + return false; + } + const auto &config_params = config.hnsw_index_params().value(); + const auto &header_params = header.get_hnsw_index_params().value(); + if ((config_params.max_links_per_node() != header_params.max_links_per_node()) || + (config_params.distance_metric() != header_params.distance_metric())) { + return false; + } + return true; +} + } void @@ -152,6 +167,7 @@ DenseTensorAttribute::onLoad() return false; } bool has_index_file = LoadUtils::file_exists(*this, DenseTensorAttributeSaver::index_file_suffix()); + bool use_index_file = has_index_file && _index && can_use_index_save_file(getConfig(), search::attribute::AttributeHeader::extractTags(tensorReader.getDatHeader())); setCreateSerialNum(tensorReader.getCreateSerialNum()); assert(tensorReader.getVersion() == DENSE_TENSOR_ATTRIBUTE_VERSION); @@ -165,7 +181,7 @@ DenseTensorAttribute::onLoad() auto raw = _denseTensorStore.allocRawBuffer(); tensorReader.readTensor(raw.data, _denseTensorStore.getBufSize()); _refVector.push_back(raw.ref); - if (_index && !has_index_file) { + if (_index && !use_index_file) { // This ensures that get_vector() (via getTensor()) is able to find the newly added tensor. setCommittedDocIdLimit(lid + 1); _index->add_document(lid); @@ -176,7 +192,7 @@ DenseTensorAttribute::onLoad() } setNumDocs(numDocs); setCommittedDocIdLimit(numDocs); - if (_index && has_index_file) { + if (_index && use_index_file) { auto buffer = LoadUtils::loadFile(*this, DenseTensorAttributeSaver::index_file_suffix()); if (!_index->load(*buffer)) { return false; -- cgit v1.2.3 From f1b9f2a373f4489a58ec382811d61a36ed72d7ec Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 24 Apr 2020 16:09:24 +0200 Subject: Clear outdated target and cleanup --- .../hosted/provision/applications/Application.java | 10 ------- .../hosted/provision/applications/Cluster.java | 16 +++-------- .../hosted/provision/autoscale/Autoscaler.java | 11 +++++--- .../maintenance/AutoscalingMaintainer.java | 33 +++++++++++----------- .../maintenance/ScalingSuggestionsMaintainer.java | 9 ++---- .../provisioning/NodeRepositoryProvisioner.java | 2 +- .../provision/testutils/MockNodeRepository.java | 8 +++--- .../provision/autoscale/AutoscalingTest.java | 22 +++++++-------- .../provision/autoscale/AutoscalingTester.java | 28 +++++++++--------- 9 files changed, 60 insertions(+), 79 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java index 05aeae45ad9..3a3d3e4ec2e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -62,16 +62,6 @@ public class Application { return with(cluster); } - /** - * Returns an application with the given target for the given cluster, - * if it exists and the target is within the bounds - */ - public Application withClusterTarget(ClusterSpec.Id id, ClusterResources target) { - Cluster cluster = clusters.get(id); - if (cluster == null) return this; - return with(cluster.withTarget(target)); - } - @Override public int hashCode() { return id.hashCode(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 9e0a9f368a8..15a5545bc2c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -64,20 +64,12 @@ public class Cluster { return new Cluster(id, min, max, suggested, target); } - public Cluster withSuggested(ClusterResources suggested) { - return new Cluster(id, min, max, Optional.of(suggested), target); - } - - public Cluster withoutSuggested() { - return new Cluster(id, min, max, Optional.empty(), target); - } - - public Cluster withTarget(ClusterResources target) { - return new Cluster(id, min, max, suggested, Optional.of(target)); + public Cluster withSuggested(Optional suggested) { + return new Cluster(id, min, max, suggested, target); } - public Cluster withoutTarget() { - return new Cluster(id, min, max, suggested, Optional.empty()); + public Cluster withTarget(Optional target) { + return new Cluster(id, min, max, suggested, target); } public NodeResources capAtLimits(NodeResources resources) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 14d1dca3e81..6dca9c9a796 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -62,8 +62,10 @@ public class Autoscaler { * @param clusterNodes the list of all the active nodes in a cluster * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time */ - public Optional suggest(Cluster cluster, List clusterNodes) { - return autoscale(cluster, clusterNodes, false); + public Optional suggest(Cluster cluster, List clusterNodes) { + return autoscale(cluster, clusterNodes, false) + .map(AllocatableClusterResources::toAdvertisedClusterResources); + } /** @@ -72,9 +74,10 @@ public class Autoscaler { * @param clusterNodes the list of all the active nodes in a cluster * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time */ - public Optional autoscale(Cluster cluster, List clusterNodes) { + public Optional autoscale(Cluster cluster, List clusterNodes) { if (cluster.minResources().equals(cluster.maxResources())) return Optional.empty(); // Shortcut - return autoscale(cluster, clusterNodes, true); + return autoscale(cluster, clusterNodes, true) + .map(AllocatableClusterResources::toAdvertisedClusterResources); } private Optional autoscale(Cluster cluster, List clusterNodes, boolean respectLimits) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 3b97195ad2d..94487547fab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -3,13 +3,16 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.collections.Pair; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.NodeResources; import com.yahoo.jdisc.Metric; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; @@ -66,26 +69,22 @@ public class AutoscalingMaintainer extends Maintainer { List clusterNodes, MaintenanceDeployment deployment) { Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)); - Cluster cluster = application.clusters().get(clusterId); - if (cluster == null) return; - if (cluster.minResources().equals(cluster.maxResources())) return; - Optional target = autoscaler.autoscale(cluster, clusterNodes); - if (target.isEmpty()) return; // current resources are fine - - logAutoscaling(target.get(), applicationId, clusterId, clusterNodes); - autoscaleTo(target.get(), clusterId, application, deployment); + Optional cluster = application.cluster(clusterId); + if (cluster.isEmpty()) return; + Optional target = autoscaler.autoscale(cluster.get(), clusterNodes); + if ( ! cluster.get().targetResources().equals(target)) + applications().put(application.with(cluster.get().withTarget(target)), deployment.applicationLock().get()); + if (target.isPresent()) { + logAutoscaling(target.get(), applicationId, clusterId, clusterNodes); + deployment.activate(); + } } - private void autoscaleTo(AllocatableClusterResources target, - ClusterSpec.Id clusterId, - Application application, - MaintenanceDeployment deployment) { - nodeRepository().applications().put(application.withClusterTarget(clusterId, target.toAdvertisedClusterResources()), - deployment.applicationLock().get()); - deployment.activate(); + private Applications applications() { + return nodeRepository().applications(); } - private void logAutoscaling(AllocatableClusterResources target, + private void logAutoscaling(ClusterResources target, ApplicationId application, ClusterSpec.Id clusterId, List clusterNodes) { @@ -93,7 +92,7 @@ public class AutoscalingMaintainer extends Maintainer { ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); log.info("Autoscaling " + application + " " + clusterType + " " + clusterId + ":" + "\nfrom " + toString(clusterNodes.size(), currentGroups, clusterNodes.get(0).flavor().resources()) + - "\nto " + toString(target.nodes(), target.groups(), target.advertisedResources())); + "\nto " + toString(target.nodes(), target.groups(), target.nodeResources())); } private String toString(int nodes, int groups, NodeResources resources) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index 985c5c7d1a7..6296543aea4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -60,22 +60,19 @@ public class ScalingSuggestionsMaintainer extends Maintainer { Application application = applications().get(applicationId).orElse(new Application(applicationId)); Optional cluster = application.cluster(clusterId); if (cluster.isEmpty()) return; - Optional suggestion = autoscaler.suggest(cluster.get(), clusterNodes); + Optional suggestion = autoscaler.suggest(cluster.get(), clusterNodes); try (Mutex lock = nodeRepository().lock(applicationId)) { applications().get(applicationId).ifPresent(a -> storeSuggestion(suggestion, clusterId, a, lock)); } } - private void storeSuggestion(Optional suggestion, + private void storeSuggestion(Optional suggestion, ClusterSpec.Id clusterId, Application application, Mutex lock) { Optional cluster = application.cluster(clusterId); if (cluster.isEmpty()) return; - if (suggestion.isPresent()) - applications().put(application.with(cluster.get().withSuggested(suggestion.get().toAdvertisedClusterResources())), lock); - else - applications().put(application.with(cluster.get().withoutSuggested()), lock); + applications().put(application.with(cluster.get().withSuggested(suggestion)), lock); } private Map> nodesByCluster(List applicationNodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 1d0ed788b41..e3b7c62cc0d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -129,7 +129,7 @@ public class NodeRepositoryProvisioner implements Provisioner { /** * Returns the target cluster resources, a value between the min and max in the requested capacity, - * and updates the application store with the received min and max, + * and updates the application store with the received min and max. */ private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec.Id clusterId, Capacity requested) { try (Mutex lock = nodeRepository.lock(applicationId)) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 0cafe35ee52..6fafd496174 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -154,10 +154,10 @@ public class MockNodeRepository extends NodeRepository { null); Application app1 = applications().get(app1Id).get(); Cluster cluster1 = app1.cluster(cluster1Id.id()).get(); - cluster1 = cluster1.withSuggested(new ClusterResources(6, 2, - new NodeResources(3, 20, 100, 1))); - cluster1 = cluster1.withTarget(new ClusterResources(4, 1, - new NodeResources(3, 16, 100, 1))); + cluster1 = cluster1.withSuggested(Optional.of(new ClusterResources(6, 2, + new NodeResources(3, 20, 100, 1)))); + cluster1 = cluster1.withTarget(Optional.of(new ClusterResources(4, 1, + new NodeResources(3, 16, 100, 1)))); try (Mutex lock = lock(app1Id)) { applications().put(app1.with(cluster1), lock); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 4dcbcd10111..f98d55511ec 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -50,9 +50,9 @@ public class AutoscalingTest { assertTrue("Too few measurements -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); tester.addMeasurements(Resource.cpu, 0.25f, 1f, 60, application1); - AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", - 15, 1, 1.3, 28.6, 28.6, - tester.autoscale(application1, cluster1.id(), min, max)); + ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", + 15, 1, 1.3, 28.6, 28.6, + tester.autoscale(application1, cluster1.id(), min, max)); tester.deploy(application1, cluster1, scaledResources); assertTrue("Cluster in flux -> No further change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); @@ -90,11 +90,11 @@ public class AutoscalingTest { new NodeResources(1, 1, 1, 1, NodeResources.DiskSpeed.any)); ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)); - AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", - 15, 1, 1.3, 28.6, 28.6, - tester.autoscale(application1, cluster1.id(), min, max)); + ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", + 15, 1, 1.3, 28.6, 28.6, + tester.autoscale(application1, cluster1.id(), min, max)); assertEquals("Disk speed from min/max is used", - NodeResources.DiskSpeed.any, scaledResources.realResources().diskSpeed()); + NodeResources.DiskSpeed.any, scaledResources.nodeResources().diskSpeed()); tester.deploy(application1, cluster1, scaledResources); tester.nodeRepository().getNodes(application1).stream() .allMatch(n -> n.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.any); @@ -115,9 +115,9 @@ public class AutoscalingTest { tester.deploy(application1, cluster1, 5, 1, resources); tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); - AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since cpu usage is too high", - 7, 1, 2.6, 80.0, 80.0, - tester.autoscale(application1, cluster1.id(), min, max)); + ClusterResources scaledResources = tester.assertResources("Scaling up since cpu usage is too high", + 7, 1, 2.6, 80.0, 80.0, + tester.autoscale(application1, cluster1.id(), min, max)); tester.deploy(application1, cluster1, scaledResources); tester.deactivateRetired(application1, cluster1, scaledResources); @@ -295,7 +295,7 @@ public class AutoscalingTest { tester.deploy(application1, cluster1, 5, 1, new NodeResources(3, 103, 100, 1)); tester.addMeasurements(Resource.memory, 0.9f, 0.6f, 120, application1); - AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high.", + ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high.", 8, 1, 3, 83, 34.3, tester.autoscale(application1, cluster1.id(), min, max)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 6eb3aaea66e..49bb51c1d79 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -79,8 +79,8 @@ class AutoscalingTester { return ClusterSpec.request(type, ClusterSpec.Id.from(clusterId)).vespaVersion("7").build(); } - public void deploy(ApplicationId application, ClusterSpec cluster, AllocatableClusterResources resources) { - deploy(application, cluster, resources.nodes(), resources.groups(), resources.advertisedResources()); + public void deploy(ApplicationId application, ClusterSpec cluster, ClusterResources resources) { + deploy(application, cluster, resources.nodes(), resources.groups(), resources.nodeResources()); } public List deploy(ApplicationId application, ClusterSpec cluster, int nodes, int groups, NodeResources resources) { @@ -101,7 +101,7 @@ class AutoscalingTester { nodeRepository().setReady(List.of(host), Agent.system, getClass().getSimpleName()); } - public void deactivateRetired(ApplicationId application, ClusterSpec cluster, AllocatableClusterResources resources) { + public void deactivateRetired(ApplicationId application, ClusterSpec cluster, ClusterResources resources) { try (Mutex lock = nodeRepository().lock(application)){ for (Node node : nodeRepository().getNodes(application, Node.State.active)) { if (node.allocation().get().membership().retired()) @@ -154,7 +154,7 @@ class AutoscalingTester { } } - public Optional autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, + public Optional autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, ClusterResources min, ClusterResources max) { Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)) .withClusterLimits(clusterId, min, max); @@ -163,7 +163,7 @@ class AutoscalingTester { nodeRepository().getNodes(applicationId, Node.State.active)); } - public Optional suggest(ApplicationId applicationId, ClusterSpec.Id clusterId, + public Optional suggest(ApplicationId applicationId, ClusterSpec.Id clusterId, ClusterResources min, ClusterResources max) { Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)) .withClusterLimits(clusterId, min, max); @@ -172,18 +172,18 @@ class AutoscalingTester { nodeRepository().getNodes(applicationId, Node.State.active)); } - public AllocatableClusterResources assertResources(String message, + public ClusterResources assertResources(String message, int nodeCount, int groupCount, double approxCpu, double approxMemory, double approxDisk, - Optional actualResources) { + Optional resources) { double delta = 0.0000000001; - assertTrue(message, actualResources.isPresent()); - assertEquals("Node count: " + message, nodeCount, actualResources.get().nodes()); - assertEquals("Group count: " + message, groupCount, actualResources.get().groups()); - assertEquals("Cpu: " + message, approxCpu, Math.round(actualResources.get().advertisedResources().vcpu() * 10) / 10.0, delta); - assertEquals("Memory: " + message, approxMemory, Math.round(actualResources.get().advertisedResources().memoryGb() * 10) / 10.0, delta); - assertEquals("Disk: " + message, approxDisk, Math.round(actualResources.get().advertisedResources().diskGb() * 10) / 10.0, delta); - return actualResources.get(); + assertTrue(message, resources.isPresent()); + assertEquals("Node count: " + message, nodeCount, resources.get().nodes()); + assertEquals("Group count: " + message, groupCount, resources.get().groups()); + assertEquals("Cpu: " + message, approxCpu, Math.round(resources.get().nodeResources().vcpu() * 10) / 10.0, delta); + assertEquals("Memory: " + message, approxMemory, Math.round(resources.get().nodeResources().memoryGb() * 10) / 10.0, delta); + assertEquals("Disk: " + message, approxDisk, Math.round(resources.get().nodeResources().diskGb() * 10) / 10.0, delta); + return resources.get(); } public ManualClock clock() { -- cgit v1.2.3 From 004de472cb3c54fb7fcc1fc8e798ee58c11be778 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 24 Apr 2020 22:06:56 +0200 Subject: Add default build settings for Ubuntu 20.04 LTS. --- default_build_settings.cmake | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/default_build_settings.cmake b/default_build_settings.cmake index c74a470ae43..66fcf1b2e16 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -88,6 +88,13 @@ function(setup_vespa_default_build_settings_ubuntu_19_10) set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) endfunction() +function(setup_vespa_default_build_settings_ubuntu_20_04) + message("-- Setting up default build settings for ubuntu 20.04") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/lib/llvm-10/lib" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/lib/llvm-10/include" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) +endfunction() + function(vespa_use_default_vespa_unprivileged) if(NOT DEFINED VESPA_UNPRIVILEGED) message("-- Setting VESPA_UNPRIVILEGED to yes") @@ -171,6 +178,8 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_fedora_33() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 19.10") setup_vespa_default_build_settings_ubuntu_19_10() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 20.04") + setup_vespa_default_build_settings_ubuntu_20_04() else() message(FATAL_ERROR "-- Unkonwn vespa build platform ${VESPA_OS_DISTRO_COMBINED}") endif() -- cgit v1.2.3 From 4e6825b47270804a36e7d76b873ca6c2fe360eac Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 13:03:16 +0200 Subject: Import java.util.logging.Level instead of com.yahoo.log.LogLevel --- .../instanceproviderservice/ConfigserverSslContextFactoryProvider.java | 2 +- .../identitydocument/IdentityDocumentResource.java | 2 +- .../instanceconfirmation/InstanceConfirmationResource.java | 2 +- .../instanceconfirmation/InstanceRefreshResource.java | 2 +- .../instanceproviderservice/instanceconfirmation/InstanceValidator.java | 2 +- .../yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java | 2 +- .../clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java | 2 +- .../java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java | 2 +- .../src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java | 2 +- .../java/com/yahoo/vespa/clustercontroller/core/FleetController.java | 2 +- .../com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java | 2 +- .../src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java | 2 +- .../java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java | 2 +- .../java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java | 2 +- .../com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java | 2 +- .../yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java | 2 +- .../yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java | 2 +- .../yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java | 2 +- .../clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java | 2 +- .../com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java | 2 +- .../main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java | 2 +- .../java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java | 2 +- .../clustercontroller/core/status/statuspage/StatusPageServer.java | 2 +- .../test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java | 2 +- .../src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java | 2 +- .../com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java | 2 +- .../java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java | 2 +- .../test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java | 2 +- .../test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java | 2 +- .../java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java | 2 +- .../java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java | 2 +- .../vespa/clustercontroller/standalone/StandAloneClusterController.java | 2 +- .../clustercontroller/utils/staterestapi/server/RestApiHandler.java | 2 +- .../src/main/java/com/yahoo/config/application/OverrideProcessor.java | 2 +- .../src/main/java/com/yahoo/config/application/PropertiesProcessor.java | 2 +- .../src/main/java/com/yahoo/config/application/Xml.java | 2 +- .../yahoo/config/model/application/provider/FilesApplicationFile.java | 2 +- .../config/model/application/provider/FilesApplicationPackage.java | 2 +- .../com/yahoo/config/model/application/provider/SchemaValidators.java | 2 +- .../config/model/application/provider/StaticConfigDefinitionRepo.java | 2 +- config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java | 2 +- .../src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java | 2 +- .../java/com/yahoo/config/model/producer/AbstractConfigProducer.java | 2 +- config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java | 2 +- .../yahoo/vespa/model/application/validation/RankSetupValidator.java | 2 +- .../src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java | 2 +- .../java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java | 2 +- .../java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java | 2 +- .../java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java | 2 +- .../src/main/java/com/yahoo/vespa/model/content/StorageGroup.java | 2 +- config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java | 2 +- .../main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java | 2 +- .../main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java | 2 +- .../src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java | 2 +- .../main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java | 2 +- .../src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java | 2 +- .../main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java | 2 +- .../java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java | 2 +- .../vespa/config/proxy/filedistribution/CachedFilesMaintainer.java | 2 +- .../vespa/config/proxy/filedistribution/FileDistributionRpcServer.java | 2 +- .../com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java | 2 +- .../yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java | 2 +- .../main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java | 2 +- .../main/java/com/yahoo/vespa/config/server/ApplicationRepository.java | 2 +- .../main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java | 2 +- .../src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java | 2 +- .../src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java | 2 +- .../java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java | 2 +- .../java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java | 2 +- .../java/com/yahoo/vespa/config/server/application/Application.java | 2 +- .../config/server/application/CompressedApplicationInputStream.java | 2 +- .../yahoo/vespa/config/server/application/ConfigConvergenceChecker.java | 2 +- .../yahoo/vespa/config/server/application/FileDistributionStatus.java | 2 +- .../main/java/com/yahoo/vespa/config/server/application/HttpProxy.java | 2 +- .../vespa/config/server/application/PermanentApplicationPackage.java | 2 +- .../com/yahoo/vespa/config/server/application/TenantApplications.java | 2 +- .../java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java | 2 +- .../src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java | 2 +- .../main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java | 2 +- .../com/yahoo/vespa/config/server/filedistribution/FileDirectory.java | 2 +- .../vespa/config/server/filedistribution/FileDistributionImpl.java | 2 +- .../java/com/yahoo/vespa/config/server/filedistribution/FileServer.java | 2 +- .../src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java | 2 +- .../main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java | 2 +- .../java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java | 2 +- .../src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java | 2 +- .../vespa/config/server/http/SessionContentStatusListResponse.java | 2 +- .../main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java | 2 +- .../src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java | 2 +- .../main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java | 2 +- .../com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java | 2 +- .../com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java | 2 +- .../com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java | 2 +- .../yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java | 2 +- .../java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java | 2 +- .../yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java | 2 +- .../src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java | 2 +- .../yahoo/vespa/config/server/provision/HostProvisionerProvider.java | 2 +- .../java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java | 2 +- .../main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java | 2 +- .../src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java | 2 +- .../vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java | 2 +- .../java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java | 2 +- .../com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java | 2 +- .../main/java/com/yahoo/vespa/config/server/session/RemoteSession.java | 2 +- .../java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java | 2 +- .../yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java | 2 +- .../java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java | 2 +- .../java/com/yahoo/vespa/config/server/session/SessionPreparer.java | 2 +- .../com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java | 2 +- .../main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java | 2 +- .../java/com/yahoo/vespa/config/server/tenant/TenantRepository.java | 2 +- .../java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java | 2 +- .../java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java | 2 +- .../com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java | 2 +- .../java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java | 2 +- .../com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java | 2 +- .../java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java | 2 +- .../java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java | 2 +- .../main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java | 2 +- .../src/main/java/com/yahoo/container/handler/VipStatusHandler.java | 2 +- .../main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java | 2 +- .../src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java | 2 +- .../main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java | 2 +- .../src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java | 2 +- .../src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java | 2 +- .../com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java | 2 +- container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java | 2 +- .../src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java | 2 +- .../java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java | 2 +- .../src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java | 2 +- .../main/java/com/yahoo/container/jdisc/component/Deconstructor.java | 2 +- .../java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java | 2 +- .../main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java | 2 +- .../src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java | 2 +- .../src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java | 2 +- .../main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java | 2 +- .../src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java | 2 +- container-search/src/main/java/com/yahoo/search/Query.java | 2 +- .../src/main/java/com/yahoo/search/cluster/ClusterSearcher.java | 2 +- .../src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java | 2 +- .../src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java | 2 +- .../src/main/java/com/yahoo/search/handler/SearchHandler.java | 2 +- .../src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java | 2 +- .../src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java | 2 +- .../src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java | 2 +- .../java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java | 2 +- .../src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java | 2 +- .../com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java | 2 +- .../com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java | 2 +- .../com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java | 2 +- .../vespa/hosted/controller/certificate/EndpointCertificateManager.java | 2 +- .../com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java | 2 +- .../yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java | 2 +- .../com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java | 2 +- .../java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java | 2 +- .../hosted/controller/maintenance/ContactInformationMaintainer.java | 2 +- .../hosted/controller/maintenance/DeploymentMetricsMaintainer.java | 2 +- .../java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java | 2 +- .../vespa/hosted/controller/maintenance/NameServiceDispatcher.java | 2 +- .../vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java | 2 +- .../vespa/hosted/controller/maintenance/RotationStatusUpdater.java | 2 +- .../yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java | 2 +- .../java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java | 2 +- .../vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java | 2 +- .../yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java | 2 +- .../hosted/controller/restapi/filter/ControllerAuthorizationFilter.java | 2 +- .../yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java | 2 +- .../hosted/controller/restapi/systemflags/SystemFlagsDeployer.java | 2 +- .../vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java | 2 +- .../java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java | 2 +- .../com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java | 2 +- .../src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java | 2 +- docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java | 2 +- docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java | 2 +- .../main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java | 2 +- .../src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java | 2 +- .../java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java | 2 +- .../main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java | 2 +- .../src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java | 2 +- .../src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java | 2 +- .../src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java | 2 +- docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java | 2 +- .../src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java | 2 +- documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java | 2 +- documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java | 2 +- .../java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java | 2 +- .../documentapi/messagebus/MessageBusVisitorDestinationSession.java | 2 +- .../java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java | 2 +- .../documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java | 2 +- .../com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java | 2 +- .../com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java | 2 +- .../yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java | 2 +- .../java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java | 2 +- .../com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java | 2 +- .../com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java | 2 +- .../java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java | 2 +- .../java/com/yahoo/vespa/filedistribution/CompressedFileReference.java | 2 +- .../src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java | 2 +- .../src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java | 2 +- .../java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java | 2 +- .../src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java | 2 +- flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java | 2 +- .../jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java | 2 +- .../jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java | 2 +- .../src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java | 2 +- .../src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java | 2 +- .../main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java | 2 +- .../java/com/yahoo/messagebus/shared/SharedIntermediateSession.java | 2 +- .../src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java | 2 +- .../src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java | 2 +- .../main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java | 2 +- logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java | 2 +- logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java | 2 +- logserver/src/main/java/com/yahoo/logserver/Flusher.java | 2 +- logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java | 2 +- .../src/main/java/com/yahoo/logserver/filter/LogFilterManager.java | 2 +- logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java | 2 +- logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java | 2 +- .../main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java | 2 +- .../src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java | 2 +- .../java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java | 2 +- .../src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java | 2 +- .../java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java | 2 +- .../src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java | 2 +- messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java | 2 +- messagebus/src/main/java/com/yahoo/messagebus/Messenger.java | 2 +- messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java | 2 +- messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java | 2 +- messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java | 2 +- .../src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java | 2 +- .../src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java | 2 +- messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java | 2 +- .../metricsproxy/http/application/ApplicationMetricsRetriever.java | 2 +- .../main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java | 2 +- .../src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java | 2 +- .../java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java | 2 +- .../src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java | 2 +- .../src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java | 2 +- .../src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java | 2 +- .../java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java | 2 +- .../yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java | 2 +- .../yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java | 2 +- .../node/admin/maintenance/identity/AthenzCredentialsMaintainer.java | 2 +- .../yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java | 2 +- .../java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java | 2 +- .../vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java | 2 +- .../vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java | 2 +- .../vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java | 2 +- .../yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java | 2 +- .../vespa/hosted/provision/maintenance/CapacityReportMaintainer.java | 2 +- .../vespa/hosted/provision/maintenance/InfrastructureProvisioner.java | 2 +- .../yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java | 2 +- .../yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java | 2 +- .../java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java | 2 +- .../yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java | 2 +- .../com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java | 2 +- .../yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java | 2 +- .../vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java | 2 +- .../com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java | 2 +- .../vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java | 2 +- .../src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java | 2 +- .../main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java | 2 +- .../vespa/orchestrator/resources/ApplicationSuspensionResource.java | 2 +- .../main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java | 2 +- .../com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java | 2 +- .../java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java | 2 +- .../java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java | 2 +- .../main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java | 2 +- .../java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java | 2 +- .../src/main/java/com/yahoo/vespa/service/duper/DuperModel.java | 2 +- .../src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java | 2 +- .../src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java | 2 +- .../main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java | 2 +- .../main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java | 2 +- .../java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java | 2 +- simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java | 2 +- simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java | 2 +- simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java | 2 +- statistics/src/main/java/com/yahoo/statistics/Value.java | 2 +- .../athenz/identityprovider/client/AthenzIdentityProviderImpl.java | 2 +- .../main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java | 2 +- .../src/main/java/com/yahoo/document/restapi/resource/RestApi.java | 2 +- .../src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java | 2 +- .../src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java | 2 +- .../src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java | 2 +- .../src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java | 2 +- vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java | 2 +- .../src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java | 2 +- vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java | 2 +- .../src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java | 2 +- .../main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java | 2 +- 292 files changed, 292 insertions(+), 292 deletions(-) diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index 7dc3496adc9..54e7a4d2947 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.google.inject.Inject; import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java index 7151de9ccc9..b49a35f0506 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument; import com.google.inject.Inject; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentApi; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java index 7a4e82d134b..59616bec749 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmati import com.google.inject.Inject; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import javax.ws.rs.Consumes; import javax.ws.rs.ForbiddenException; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java index 53972a39c97..c62e5009c8f 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmati import com.google.inject.Inject; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import javax.ws.rs.Consumes; import javax.ws.rs.ForbiddenException; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index 3a926922f2b..b1958bf5eed 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -7,7 +7,7 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index 466a9783bb3..dfbad290fbb 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -7,7 +7,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.http.servlet.ServletRequest; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java index 7f0acd83b0d..ac5a7e3ed53 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.clustercontroller.apps.clustercontroller; import com.google.inject.Inject; import com.yahoo.cloud.config.ClusterInfoConfig; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.apputil.communication.http.JDiscHttpRequestHandler; import com.yahoo.vespa.clustercontroller.core.restapiv2.ClusterControllerStateRestAPI; import com.yahoo.vespa.clustercontroller.utils.staterestapi.server.RestApiHandler; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java index 62cd158c759..0fcb5b65237 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java index 964e7c5e947..e0f3b294c4d 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index d943cf27f9c..82fb4c825b9 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.document.FixedBucketSpaces; import com.yahoo.exception.ExceptionUtils; import com.yahoo.jrt.ListenFailedException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java index 9d23eead605..205a6e13242 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; import java.util.Map; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java index 82d13e2d9ef..f98de8ca581 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.collections.Pair; import com.yahoo.jrt.Target; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.distribution.Group; import com.yahoo.vdslib.state.*; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java index 318768454c6..f1fe29cf0e3 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.ErrorCode; import com.yahoo.jrt.Target; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index 3c19f70d1e2..a8aa15d5959 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.Spec; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.state.*; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java index 5ecb57a1c76..55fc0294dd7 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.ErrorCode; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.*; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java index b9d02f16090..78e71d63e6a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.database; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java index 3ad0587b740..0fdfd46dec5 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java @@ -8,7 +8,7 @@ import java.util.logging.Logger; import java.util.*; import java.nio.charset.Charset; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; public class MasterDataGatherer { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java index 880f2cdaf19..453216039d9 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.clustercontroller.core.rpc.SlimeClusterStateBundleCodec; import org.apache.zookeeper.*; import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.data.ACL; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; import com.yahoo.vdslib.state.Node; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java index eecdcc75228..64ca08dc997 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.restapiv2.requests; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.time.TimeBudget; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java index c3c5c9e3b98..a2619692961 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java @@ -14,7 +14,7 @@ import com.yahoo.jrt.Values; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.State; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.core.ActivateClusterStateVersionRequest; import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; import com.yahoo.vespa.clustercontroller.core.Communicator; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index 87901a9bdb4..d368d377aee 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -15,7 +15,7 @@ import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.BackOffPolicy; import com.yahoo.jrt.slobrok.api.Register; import com.yahoo.jrt.slobrok.api.SlobrokList; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index 825e6b033cd..5af1af519bb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -8,7 +8,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.Node; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.core.NodeInfo; import com.yahoo.vespa.clustercontroller.core.NodeLookup; import com.yahoo.vespa.clustercontroller.core.Timer; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index 6e4c0bc7eaa..c7f7682a62a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.core.status.statuspage; import com.yahoo.exception.ExceptionUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.BufferedReader; import java.io.IOException; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java index b611d573ff5..8d8835d5932 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java @@ -15,7 +15,7 @@ import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.BackOffPolicy; import com.yahoo.jrt.slobrok.api.Register; import com.yahoo.jrt.slobrok.api.SlobrokList; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.*; import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator; import com.yahoo.vespa.clustercontroller.core.rpc.RPCUtil; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java index e161c4aacb7..bcf98d003a6 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.core.testutils.LogFormatter; import java.util.logging.Logger; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index 059d7d68754..d69725ebe20 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -9,7 +9,7 @@ import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.BackOffPolicy; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.state.ClusterState; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index f919bd867c6..c7f25864341 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -7,7 +7,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java index 532919f05da..07d86459c0f 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.ClusterState; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java index 209da523705..ecdbdaeb7ce 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java index 398c1c16473..695e2954546 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.junit.Test; import java.util.concurrent.TimeoutException; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java index deef3157eed..15587d6a3fb 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.testutils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.core.DummyVdsNode; diff --git a/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java b/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java index 2d6880e0551..1e163cf7460 100644 --- a/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java +++ b/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.standalone; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; import com.yahoo.vespa.clustercontroller.core.FleetController; diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java index acc4782e5eb..fc7b3a96d53 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.utils.staterestapi.server; import com.google.common.util.concurrent.UncheckedTimeoutException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.time.TimeBudget; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequestHandler; diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java index 23273f9f1bf..02862207e53 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java @@ -4,7 +4,7 @@ package com.yahoo.config.application; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.XML; import org.w3c.dom.Document; import org.w3c.dom.Element; diff --git a/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java index 65120c3677c..2966030db40 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.XML; import org.w3c.dom.*; diff --git a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java index 1cdb54a743c..afc6634b4cd 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java @@ -3,7 +3,7 @@ package com.yahoo.config.application; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.io.reader.NamedReader; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.XML; import org.w3c.dom.Document; diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java index a8e1256e032..521bc141b2b 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.config.util.ConfigUtils; diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index e93246f49e7..c2ad0817de5 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -22,7 +22,7 @@ import com.yahoo.path.Path; import com.yahoo.io.HexDump; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java index d5d890dba43..23fe0109871 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java @@ -3,7 +3,7 @@ package com.yahoo.config.model.application.provider; import com.yahoo.component.Version; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.osgi.framework.Bundle; import org.xml.sax.SAXException; diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java index 872aa7fb56f..8faef6b2ce6 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java @@ -3,7 +3,7 @@ package com.yahoo.config.model.application.provider; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.config.util.ConfigUtils; diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 4e9eb3bdb2e..2925b394c83 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -11,7 +11,7 @@ import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.graph.ModelGraphBuilder; import com.yahoo.config.model.graph.ModelNode; import com.yahoo.config.model.provision.HostsXmlProvisioner; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.XML; import com.yahoo.config.model.producer.AbstractConfigProducer; diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java index 4cd0c1815dd..107908c91db 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java @@ -3,7 +3,7 @@ package com.yahoo.config.model.builder.xml; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.XML; import org.w3c.dom.Document; import org.w3c.dom.Element; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index cce5a7850a0..43eeb613e6b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -6,7 +6,7 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.subscription.ConfigInstanceUtil; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 6f2123f248c..058b456fdb8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -30,7 +30,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstants; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index b6f7ab4ff62..71e90686e22 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.io.IOUtils; import com.yahoo.log.InvalidLogFormatException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogMessage; import com.yahoo.yolean.Exceptions; import com.yahoo.system.ProcessExecuter; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java index cb194e34bca..92785e00436 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.builder; import com.yahoo.config.model.deploy.ConfigDefinitionStore; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.producer.UserConfigRepo; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.XML; import com.yahoo.vespa.config.*; import com.yahoo.vespa.model.builder.xml.dom.DomConfigPayloadBuilder; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 1b0e04b50a8..44c3301b6a4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -7,7 +7,7 @@ import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.text.XML; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Configserver; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 804a5442608..0b0fd1897e7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -6,7 +6,7 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.admin.Admin; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index c9caca1831f..9067df53a91 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -8,7 +8,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.UserConfigRepo; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.XML; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.Affinity; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index f0c374b398f..8ff82e24760 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -6,7 +6,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index 46d41b1e0b0..901d81927a5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.utils; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.producer.UserConfigRepo; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.*; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.config.ConfigDefinition.DefaultValued; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index 0a36f06a1b9..e27c26bd248 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -12,7 +12,7 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.TargetWatcher; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.JRTMethods; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java index ac0c036774b..d9d2de2fac2 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.config.ConfigCacheKey; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java index 91bb669f396..e10289733df 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.proxy; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigCacheKey; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.RawConfig; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java index 51446882025..16b80f8d31e 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.*; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 545b962f6ff..ddc0c0d8320 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -5,7 +5,7 @@ import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; import com.yahoo.vespa.config.RawConfig; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index 2a33e8c6928..ddf3bc4d9ce 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigCacheKey; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.TimingValues; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java index d8a8c5ce941..c7567c45c27 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java @@ -5,7 +5,7 @@ import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.config.subscription.impl.GenericConfigHandle; import com.yahoo.config.subscription.impl.GenericConfigSubscriber; import com.yahoo.config.subscription.impl.JRTConfigRequester; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.TimingValues; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java index eec045cdb0a..c2daa6e739b 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.proxy.filedistribution; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.filedistribution.FileDownloader; import java.io.File; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java index 7db9761d86a..6f5b125cc1e 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Request; import com.yahoo.jrt.StringArray; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceDownload; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java index 47f478ea4d7..6b0be5bca16 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.proxy.filedistribution; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.File; import java.time.Instant; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java index 592f5211eed..970e4da5511 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java @@ -6,7 +6,7 @@ import com.yahoo.jrt.Method; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.defaults.Defaults; import net.jpountz.xxhash.XXHashFactory; diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java index d34b1fd949f..64cae0f31a0 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java @@ -5,7 +5,7 @@ import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.vespa.configserver.flags.FlagsDb; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 95c8733b540..8db1ac5c0a0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -24,7 +24,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 2a426e4cccc..a3fbe9ec5fb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.TransientException; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.state.StateMonitor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; import com.yahoo.yolean.Exceptions; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java index 3e9783cabe9..72d91e99278 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java @@ -5,7 +5,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.ConfigDefinitionDir; import com.yahoo.config.model.application.provider.Bundle; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.defaults.Defaults; import java.io.File; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index d46c551fa7e..bc022a3342c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -11,7 +11,7 @@ import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.Zone; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.GenerationCounter; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.model.SuperModelConfigProvider; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index 51c5be1b1f0..aa6c2c1a2b2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -7,7 +7,7 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.component.Version; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java index 30c68b3830c..2a632a75330 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server; import com.google.common.base.Splitter; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index c8f3bcd5b3f..ef9c5161e98 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -6,7 +6,7 @@ import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigCacheKey; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigKey; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index c94f739b958..931d85dbe45 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.application; import com.google.common.io.ByteStreams; import com.google.common.io.Files; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index e841dd9b8d9..9ffca3d5ea5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -8,7 +8,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.http.JSONResponse; import org.glassfish.jersey.client.ClientProperties; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java index f37225a7e7b..f00acacef03 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.http.JSONResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java index c1d29329594..88b13662b7f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java @@ -6,7 +6,7 @@ import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpFetcher; import com.yahoo.vespa.config.server.http.NotFoundException; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java index b64aa1c771c..52aae860480 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.cloud.config.ConfigserverConfig; import static com.yahoo.vespa.defaults.Defaults.getDefaults; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 7ff5d41485d..a1b7df5f19d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.transaction.Transaction; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index eaaa4f93bd1..3e5c1f51990 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.tenant.TenantRepository; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 4bd5cf30cc6..5f2bece849b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.ActivationConflictException; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 09f5178cf6b..825a0b4a631 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -12,7 +12,7 @@ import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.reader.NamedReader; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index f710eec27ba..25747113d9e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -5,7 +5,7 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import net.jpountz.xxhash.XXHash64; import net.jpountz.xxhash.XXHashFactory; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java index 2178af6915d..3040a43bde4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Spec; import com.yahoo.jrt.StringArray; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.defaults.Defaults; import java.io.File; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index cb4945bdab3..465c8a02200 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java index a7f8b8164a5..cb33371aee9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java @@ -7,7 +7,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import com.google.common.collect.Collections2; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * A host registry with a mapping between hosts (hostname as a String) and some type T diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java index 6559292645c..52fe57a7923 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java index 0397678fd41..592aed83024 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java @@ -5,7 +5,7 @@ import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.tenant.TenantRepository; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index 7cdb8d3a60c..78bc0f48232 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.vespa.config.server.ActivationConflictException; import com.yahoo.yolean.Exceptions; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java index d69c0084180..7fc9c4ead47 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java index 76a022df580..28dfa30e7ac 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.http; import ai.vespa.util.http.VespaHttpClientBuilder; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.apache.http.HttpEntity; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java index 2d5cce2d4f1..577a0fdd17a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.http; import ai.vespa.util.http.VespaHttpClientBuilder; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java index d8385456866..ddeb49fda6f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -9,7 +9,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingMatch; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.host.HostRegistry; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java index 49c0cb7427f..5b4c32da7e1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.tenant.TenantRepository; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index dc2c23dc9a3..5289092e9ce 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -12,7 +12,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java index 4f64b73b403..15ed5fd2408 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.server.metrics; import ai.vespa.util.http.VespaHttpClientBuilder; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 99bec2c9db7..e4276687c38 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -15,7 +15,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ServerCache; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index cc4378ae05e..2662e14e8e9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -16,7 +16,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.TransientException; import com.yahoo.config.provision.Zone; import com.yahoo.lang.SettableOptional; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index 6d0403f21c8..032e7232744 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -18,7 +18,7 @@ import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.component.Version; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.host.HostValidator; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java index 249c8639ea9..fd840a6385a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.statistics.Statistics; import com.yahoo.statistics.Counter; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java index 32a641a3bc7..eb14947f73f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java @@ -5,7 +5,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.provision.Provisioner; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.Optional; import java.util.logging.Logger; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java index f58c6cfdee5..03b11a89d08 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.jrt.Target; import com.yahoo.jrt.TargetWatcher; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.server.GetConfigContext; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 48580dbc6f4..ff39faa9d29 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -6,7 +6,7 @@ import com.yahoo.collections.Pair; import com.yahoo.config.provision.TenantName; import com.yahoo.component.Version; import com.yahoo.jrt.Request; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ErrorCode; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 851ad166c2d..f2cabe77f36 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -21,7 +21,7 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ErrorCode; import com.yahoo.vespa.config.JRTMethods; import com.yahoo.vespa.config.protocol.ConfigResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java index 8d17a32d102..4a030de8c26 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.security.NodeIdentifierException; import com.yahoo.config.provision.security.NodeIdentity; import com.yahoo.jrt.Request; import com.yahoo.jrt.SecurityContext; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.tls.MixedMode; import com.yahoo.security.tls.TransportSecurityUtils; import com.yahoo.vespa.config.ConfigKey; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java index ad7a5116ac4..e697e7193ca 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.GlobalComponentRegistry; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java index 3ac98f9ac4c..6a27a300a15 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.recipes.cache.ChildData; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 39c15474f0e..de084e598c9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -5,7 +5,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.lang.SettableOptional; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ReloadHandler; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index 936b9bdefda..6b5ad0c10a8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -6,7 +6,7 @@ import com.google.common.collect.Multiset; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ReloadHandler; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java index a3fad3d7322..1f11c0f9737 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index 5ae1289033d..9473855e2aa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -6,7 +6,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.application.provider.*; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.*; import com.yahoo.config.provision.ApplicationId; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 6c77964a58b..d39ddd68f25 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -22,7 +22,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.lang.SettableOptional; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.application.ApplicationSet; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 81777bf3642..bd130ebb153 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.transaction.NestedTransaction; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java index 9842c90fff1..a1de394f61a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.TreeCache; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 2e09d830783..b82574efa9f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -7,7 +7,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java index 2f9a5eb9277..6f0638a4760 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java @@ -6,7 +6,7 @@ import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.FileReference; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java index f07327517f8..748223b13a3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.zookeeper; import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.Curator; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java index 773888a8a03..fdc96ea390e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.zookeeper; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.curator.recipes.CuratorCounter; import java.util.ArrayList; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java index ad0d4a2e540..76a934236d0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.path.Path; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.util.ConfigUtils; import java.io.*; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java index d321edefe67..3ec515a1528 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index fa0bbac05c8..c5015536e03 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -17,7 +17,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 66891e54fb9..287996f6bec 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -20,7 +20,7 @@ import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index d87b38e8b18..1d4a174dc55 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -20,7 +20,7 @@ import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.osgi.OsgiImpl; import com.yahoo.osgi.OsgiWrapper; import com.yahoo.statistics.Statistics; diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java index eceffb379aa..f2df5ddb7c5 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java @@ -18,7 +18,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.defaults.Defaults; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java b/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java index dc3bd6fe8a8..dde4e8b73e4 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java @@ -5,7 +5,7 @@ import com.yahoo.io.BufferChain; import com.yahoo.io.WritableByteTransmitter; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import java.io.IOException; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index 62dc81aa103..8371acb59d4 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -11,7 +11,7 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import java.io.IOException; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index c58d49bf8c8..9e69b9890b5 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -9,7 +9,7 @@ import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.UnsafeContentInputStream; import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index 99732af9d31..a34ceb7f613 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -15,7 +15,7 @@ import com.yahoo.jdisc.handler.OverloadException; import com.yahoo.jdisc.handler.ReadableContentChannel; import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.net.URI; import java.time.Duration; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index faa08402cdc..29691668975 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -7,7 +7,7 @@ import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.jdisc.Timer; import com.yahoo.jdisc.application.MetricConsumer; import com.yahoo.jdisc.core.SystemTimer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.Map; import java.util.TreeSet; diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java index a18d1eee0fc..3a1eb216d42 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java @@ -7,7 +7,7 @@ import com.google.common.util.concurrent.SettableFuture; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.processing.Request; import com.yahoo.processing.Response; import com.yahoo.processing.execution.Execution; diff --git a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java index f261c7611f1..d76804a1d5f 100644 --- a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.IOException; import java.io.OutputStream; diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java index 507ef6952b6..dce791d375f 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.container.jdisc; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java index d07aefef75b..e0cb69d8499 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java @@ -15,7 +15,7 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.container.di.componentgraph.cycle.CycleFinder; import com.yahoo.container.di.componentgraph.cycle.Graph; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigKey; import java.lang.annotation.Annotation; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index e174de19394..81a2c8a9373 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -37,7 +37,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.Register; import com.yahoo.jrt.slobrok.api.SlobrokList; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.messagebus.network.rpc.SlobrokConfigSubscriber; import com.yahoo.net.HostName; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java index 6a46e331762..1f111b8958c 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java @@ -6,7 +6,7 @@ import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.container.di.ComponentDeconstructor; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.jdisc.SharedResource; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java index 1ab418fd523..5da472b9b28 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java @@ -5,7 +5,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.jdisc.service.CurrentContainer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.IntermediateSessionParams; import com.yahoo.messagebus.jdisc.MbusServer; import com.yahoo.messagebus.shared.SharedIntermediateSession; diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index 7193433ccf7..ef9b4e8d1ca 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -12,7 +12,7 @@ import com.yahoo.jdisc.ReferencedResource; import com.yahoo.jdisc.References; import com.yahoo.jdisc.ResourceReference; import com.yahoo.jdisc.SharedResource; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.ConfigAgent; import com.yahoo.messagebus.DynamicThrottlePolicy; import com.yahoo.messagebus.IntermediateSessionParams; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 329a9caaf91..10ecb8604ea 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -2,7 +2,7 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.Inspector; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java index 8020088c2e3..c24a2824b95 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.query.Substring; import java.util.*; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 7279e1d6a3d..5b8f7b8b106 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -9,7 +9,7 @@ import com.yahoo.language.Language; import com.yahoo.language.Linguistics; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.StemList; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.*; diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index ebea2f95af7..8e83782c9c7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -5,7 +5,7 @@ import com.yahoo.component.chain.dependencies.Before; import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.container.protect.Error; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.metrics.simple.MetricSettings; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.processing.request.CompoundName; diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index dc8e2b70740..0966ae3e4c0 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -7,7 +7,7 @@ import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.fs4.MapEncoder; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.QueryException; diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 645c6446ef1..3afeb35ad2d 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -3,7 +3,7 @@ package com.yahoo.search.cluster; import com.yahoo.component.ComponentId; import com.yahoo.container.protect.Error; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.yolean.Exceptions; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java index 8280917ca2f..d30464e349e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java @@ -3,7 +3,7 @@ package com.yahoo.search.grouping; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index 7c2e774f68b..30860f9570b 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -13,7 +13,7 @@ import java.util.logging.Logger; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Provides; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.QueryCanonicalizer; diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 5e3b79c1545..ee81f2c819e 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -18,7 +18,7 @@ import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.net.UriTools; import com.yahoo.prelude.query.QueryException; diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java index b73ef1298ee..d4db21c3810 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java @@ -2,7 +2,7 @@ package com.yahoo.search.query.rewrite; import com.yahoo.fsa.FSA; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.search.Query; import com.yahoo.search.intent.model.IntentModel; import com.yahoo.search.intent.model.InterpretationNode; diff --git a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java index 9d0e110a6dd..d678b4d2487 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java @@ -2,7 +2,7 @@ package com.yahoo.search.rendering; import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.URI; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.hitfield.HitField; diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java index 018f4fc7aa9..7d9d3f147f5 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java @@ -11,7 +11,7 @@ import java.util.ListIterator; import java.util.Map; import java.util.logging.Logger; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.metrics.simple.Counter; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.prelude.query.CompositeItem; diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 4750bac551c..9ac1cb33d1b 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -5,7 +5,7 @@ import com.yahoo.document.DocumentId; import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.select.parser.TokenMgrException; import com.yahoo.fs4.DocsumPacket; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.routing.Route; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index 795b62663d5..ad92512d1a1 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -17,7 +17,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.routing.Route; diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java index 0aaf301e071..e9de02b243b 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.streamingvisitors.tracing; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.function.Supplier; import java.util.logging.Logger; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java index 384b1342ea4..b0170486eca 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.BufferedReader; import java.io.IOException; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index c3a9a8484c9..c478c396ca3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -6,7 +6,7 @@ import com.google.common.cache.CacheLoader; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index 4cb62892555..8cef88cacf0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.flags.BooleanFlag; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 893956e5767..b63b4e4fcc6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -5,7 +5,7 @@ import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index bc206011bb1..58547edc52a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -16,7 +16,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index 4e461534cc0..75a57ba49e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java index 684fb091d92..aebf4a61966 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import java.util.ArrayList; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index d1f2087ce12..8346fe7a0f9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.TenantController; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 1ce1e7f4c0c..9843d808564 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 23e3149ec1e..d78091b5116 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java index 83460d8f05a..f1b8d40cf80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index cf06489b8c1..e08ed6a6178 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.yolean.Exceptions; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java index 35131d3db80..0253cdb1e34 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index 565acf6ebd5..e678c4bb5ca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.yolean.Exceptions; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 77cdd8d342d..ef347c9be19 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -9,7 +9,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 95867b3786c..5cfd07dcb2e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 4d3cfacab14..8eccb92c2e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.restapi.Path; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index ec95f0bbb4c..757ccbe900e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -6,7 +6,7 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.role.Action; import com.yahoo.vespa.hosted.controller.api.role.Enforcer; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java index 4916b7b24a1..2e7f75cbb9c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java @@ -9,7 +9,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.KeyUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java index 9bd3b70cfab..b4fa9d261c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.systemflags; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.SystemName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.Flags; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java index 2f7716aaff0..76c746cd6c7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java @@ -6,7 +6,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.logging.AccessLog; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.JacksonJsonResponse; import com.yahoo.restapi.Path; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 7618e8210e3..41a3784163d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -6,7 +6,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ListMultimap; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 589229b32d4..926f5a352cb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index a25a15f58ca..49a101a2ed6 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -22,7 +22,7 @@ import com.github.dockerjava.core.command.PullImageResultCallback; import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory; import com.google.inject.Inject; import com.yahoo.config.provision.DockerImage; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerExecTimeoutException; diff --git a/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java b/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java index a1b640d5eb5..fd1b81fce40 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java @@ -5,7 +5,7 @@ import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; import com.yahoo.document.json.JsonWriter; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.statistics.Counter; import com.yahoo.text.Utf8; diff --git a/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java index e1bcf9c7c42..f074d4a287c 100644 --- a/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java @@ -5,7 +5,7 @@ import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentUpdate; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** *

Simple layer on top of {@link DocumentProcessor}, in order to make docproc diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index f93598058cf..caa8b0b282d 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -24,7 +24,7 @@ import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.jdisc.MbusRequest; import com.yahoo.processing.execution.chain.ChainRegistry; import com.yahoo.statistics.Statistics; diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index d1d5256efe3..383918ce188 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -9,7 +9,7 @@ import com.yahoo.docproc.DocprocService; import com.yahoo.docproc.DocumentProcessor; import com.yahoo.docproc.HandledProcessingException; import com.yahoo.docproc.Processing; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import java.io.PrintWriter; diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java index 2e63e2defff..47d639e5dc6 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java @@ -19,7 +19,7 @@ import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.RequestDispatch; import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.jdisc.MbusRequest; diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java index 702343acd1c..7c4a6c8c2c3 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java @@ -13,7 +13,7 @@ import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.TestAndSetMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import java.util.logging.Logger; diff --git a/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java index 448d5213e6d..93d87a49c64 100644 --- a/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java @@ -11,7 +11,7 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.document.datatypes.Array; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; diff --git a/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java index 18e4294480b..f6be7a4d990 100644 --- a/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java @@ -13,7 +13,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.document.datatypes.Array; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java index aa66f6c7c3c..30b7f9bf718 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java @@ -19,7 +19,7 @@ import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.indexinglanguage.AdapterFactory; import com.yahoo.vespa.indexinglanguage.SimpleAdapterFactory; diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java index 000568214cd..783a003043e 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java @@ -4,7 +4,7 @@ package com.yahoo.docprocs.indexing; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 21a163aa2b9..d5da805dcdc 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -6,7 +6,7 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java index 2bb93659a78..1842c1972df 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java @@ -9,7 +9,7 @@ import java.util.logging.Logger; import com.yahoo.document.BucketId; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.objects.BufferSerializer; /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java index c26c986152c..83a7bc20353 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java @@ -5,7 +5,7 @@ import com.yahoo.document.BucketId; import com.yahoo.document.BucketIdFactory; import com.yahoo.document.select.BucketSelector; import com.yahoo.document.select.parser.ParseException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.Map; import java.util.Set; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 33286d227bf..79df0db0d17 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -23,7 +23,7 @@ import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentReply; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.ErrorCode; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.MessageBus; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java index 7813cb30d7e..247364c8a7a 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java @@ -6,7 +6,7 @@ import com.yahoo.documentapi.VisitorDestinationParameters; import com.yahoo.documentapi.VisitorDestinationSession; import com.yahoo.documentapi.VisitorResponse; import com.yahoo.documentapi.messagebus.protocol.*; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.*; import java.util.logging.Logger; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index 54e6fbf8be1..b6e822b597e 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -18,7 +18,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.VisitorInfoMessage; import com.yahoo.documentapi.messagebus.protocol.WrongDistributionReply; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.DestinationSession; import com.yahoo.messagebus.DestinationSessionParams; import com.yahoo.messagebus.DynamicThrottlePolicy; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java index cd181ac3c51..ea8e87a8df0 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java @@ -5,7 +5,7 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.document.DocumentGet; import com.yahoo.document.select.DocumentSelector; import com.yahoo.document.select.Result; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java index 1f3810f9a02..ace19466496 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java @@ -12,7 +12,7 @@ import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentSerializer; import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Routable; import com.yahoo.vdslib.DocumentSummary; import com.yahoo.vdslib.SearchResult; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java index 3933b188dc6..c91db403cbd 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java @@ -11,7 +11,7 @@ import com.yahoo.document.serialization.DocumentSerializer; import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Routable; import java.util.ArrayList; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java index 098e0efc41c..0b9498d798a 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java @@ -2,7 +2,7 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.messagebus.routing.RoutingPolicy; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.Map; import java.util.logging.Logger; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java index 46d0cd3fc37..236a2a6c0f6 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java @@ -6,7 +6,7 @@ import com.yahoo.document.BucketId; import com.yahoo.document.BucketIdFactory; import com.yahoo.jrt.slobrok.api.IMirror; import com.yahoo.jrt.slobrok.api.Mirror; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java index 3251c038fc7..b13a273da27 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java @@ -2,7 +2,7 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.jrt.slobrok.api.Mirror; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.routing.Hop; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java index 69a5346adb5..527c94a4d2a 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.systemstate.rule; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java index bd4401d3cab..e83a78df719 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java @@ -5,7 +5,7 @@ import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.config.FileReference; import com.yahoo.jrt.*; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.concurrent.locks.Lock; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java index ddab35ba14d..5ea4092e0c6 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.filedistribution; import com.google.common.io.ByteStreams; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveInputStream; import org.apache.commons.compress.archivers.ArchiveOutputStream; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 5829ab37b77..57281d4a84b 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.filedistribution; import com.yahoo.config.FileReference; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.defaults.Defaults; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index ff9efc74929..0228ad0a5a2 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -8,7 +8,7 @@ import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Method; import com.yahoo.jrt.Request; import com.yahoo.jrt.Supervisor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import net.jpountz.xxhash.StreamingXXHash64; import net.jpountz.xxhash.XXHashFactory; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index f1d31156d79..4a3c474a436 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -6,7 +6,7 @@ import com.yahoo.config.FileReference; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java index 26cc3553c1f..0873ed57adb 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java @@ -13,7 +13,7 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import net.jpountz.xxhash.XXHash64; import net.jpountz.xxhash.XXHashFactory; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java index abe8d407ab0..0082950ffb0 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java @@ -1,7 +1,7 @@ // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags.file; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.json.FlagData; diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java index 53270f3b51e..0f69003d7ee 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java @@ -6,7 +6,7 @@ import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.athenz.RequestResourceMapper.ResourceNameAndAction; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java index 33d344c4f2a..514fdbcc745 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java @@ -11,7 +11,7 @@ import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.UncheckedIOException; import java.util.Optional; diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java index 4bd78813cba..6ecb36f7af9 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java @@ -9,7 +9,7 @@ import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.RequestDeniedException; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.service.ClientProvider; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java index bc3d1edda7c..5893e4f6252 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java @@ -10,7 +10,7 @@ import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.service.CurrentContainer; import com.yahoo.jdisc.service.ServerProvider; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.*; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.shared.ServerSession; diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java index fd377aaa7d1..d9db66b6041 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus.shared; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.ResourceReference; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.DestinationSession; import com.yahoo.messagebus.DestinationSessionParams; import com.yahoo.messagebus.EmptyReply; diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java index f543e1b5c8d..9a50101a492 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus.shared; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.ResourceReference; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java index 704a73710cf..8c084f0f9e4 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus.shared; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.jdisc.AbstractResource; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.DestinationSessionParams; import com.yahoo.messagebus.IntermediateSessionParams; import com.yahoo.messagebus.MessageBus; diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java index d49a5ea67b8..a75bf6513f9 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus.shared; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.ResourceReference; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ReplyHandler; diff --git a/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java index 9065b47fa52..39bdeaa624e 100644 --- a/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java +++ b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java @@ -3,7 +3,7 @@ package ai.vespa.logserver.protocol; import ai.vespa.logserver.protocol.protobuf.LogProtocol; import com.google.protobuf.InvalidProtocolBufferException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogMessage; import java.time.Instant; diff --git a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java index f2393edeff3..b1d8de1bd98 100644 --- a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.plugin.Plugin; import com.yahoo.plugin.SystemPropertyConfig; diff --git a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java index 32427604bd6..c67dbe3bce8 100644 --- a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.logserver.handlers.archive.ArchiverPlugin; import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin; diff --git a/logserver/src/main/java/com/yahoo/logserver/Flusher.java b/logserver/src/main/java/com/yahoo/logserver/Flusher.java index d8d18bd2070..8958d350a89 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Flusher.java +++ b/logserver/src/main/java/com/yahoo/logserver/Flusher.java @@ -7,7 +7,7 @@ import java.util.Iterator; import java.util.List; import java.util.logging.Logger; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.logserver.handlers.LogHandler; /** diff --git a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java index afc901dc9e2..e74fa7a81f0 100644 --- a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java +++ b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogMessage; import com.yahoo.logserver.handlers.LogHandler; diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java index 0e91bac164e..91f2ba3b96a 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java @@ -4,7 +4,7 @@ package com.yahoo.logserver.filter; import java.util.HashMap; import java.util.Map; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * The LogFilterManager keeps track of associations between diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java index 144d498e290..fbd192690a3 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver.filter; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.event.Count; import com.yahoo.log.event.CountGroup; import com.yahoo.log.event.Event; diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java index 776125fe24b..f1c08b9d390 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java @@ -7,7 +7,7 @@ import java.util.List; import java.util.logging.Logger; import com.yahoo.io.FatalErrorHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogMessage; import java.util.concurrent.BlockingQueue; diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index be788c23e80..e73ae531ecc 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver.handlers.archive; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogMessage; import com.yahoo.logserver.filter.LogFilter; import com.yahoo.logserver.filter.LogFilterManager; diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java index 83d6a4a0def..8635534ba7b 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java @@ -7,7 +7,7 @@ import java.io.IOException; import java.io.Writer; import java.util.logging.Logger; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * This class is not thread-safe. diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java index fbf38fadc90..dc457d81ed5 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java @@ -12,7 +12,7 @@ import java.util.ArrayList; import java.util.logging.Logger; import java.util.logging.Level; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.event.Event; import com.yahoo.logserver.filter.LevelFilter; import com.yahoo.log.LogMessage; diff --git a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java index a398c2d6aff..d594df933f6 100644 --- a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java +++ b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java @@ -5,7 +5,7 @@ import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.event.MalformedEventException; import com.yahoo.log.InvalidLogFormatException; import com.yahoo.log.LogMessage; diff --git a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java index 93160c2a7a2..982cb8a9de7 100644 --- a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java +++ b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.Values; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogMessage; import com.yahoo.logserver.LogDispatcher; import org.junit.Test; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java index 76d3bc07d2b..d2e3c84baab 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus; import com.yahoo.concurrent.SystemTimer; import com.yahoo.concurrent.Timer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; /** diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 5622aebab66..8524d7e8c06 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus; import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.concurrent.SystemTimer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.NetworkOwner; import com.yahoo.messagebus.routing.Resender; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java index 7211e4cead0..5f2ff58f3fd 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.ArrayDeque; import java.util.ArrayList; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java b/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java index 102e0be923d..d875cc32b3f 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java @@ -2,7 +2,7 @@ package com.yahoo.messagebus; import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.routing.RoutingPolicy; import com.yahoo.text.Utf8String; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java b/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java index 781235847a0..da143877083 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java @@ -4,7 +4,7 @@ package com.yahoo.messagebus; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.routing.Resender; import com.yahoo.messagebus.routing.RoutingNode; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java b/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java index 38963645d54..8fe3072cba9 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.ArrayList; import java.util.Collections; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java index 53c0c855116..9cab213b628 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java @@ -7,7 +7,7 @@ import com.yahoo.jrt.RequestWaiter; import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8Array; import java.util.LinkedList; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java index b17bb892f7d..1a6944cbae3 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java @@ -6,7 +6,7 @@ import com.yahoo.component.Vtag; import com.yahoo.jrt.Spec; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.MessageBus; import com.yahoo.messagebus.MessageBusParams; import com.yahoo.messagebus.Protocol; diff --git a/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java index b5e39035bfb..6549e06834d 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.messagebus; import com.yahoo.component.Vtag; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.network.rpc.test.TestServer; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index d20d4b0cb3c..8edae823a97 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -6,7 +6,7 @@ import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.util.http.VespaHttpClientBuilder; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java index b5bd6c073cd..449309d8a3e 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java @@ -3,7 +3,7 @@ package ai.vespa.metricsproxy.service; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.jrt.ErrorCode; import com.yahoo.jrt.Request; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index 9068be81b65..a5c0e0e2395 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -2,7 +2,7 @@ package ai.vespa.metricsproxy.service; import ai.vespa.util.http.VespaHttpClientBuilder; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java index ec36ae344bc..4bc9055c6a5 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java @@ -2,7 +2,7 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.HealthMetric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.json.JSONException; import org.json.JSONObject; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java index c8fbc83eb59..3e8eeea1268 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java @@ -3,7 +3,7 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.BufferedReader; import java.io.FileNotFoundException; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java index 9600dfca042..25e2b9068ab 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java @@ -3,7 +3,7 @@ package ai.vespa.metricsproxy.telegraf; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.system.execution.ProcessExecutor; import com.yahoo.system.execution.ProcessResult; import org.apache.velocity.VelocityContext; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java index 23da51ea082..0f36bf6125e 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java @@ -1,7 +1,7 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.telegraf; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.ArrayList; import java.util.Collections; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java index b22c76a987f..74b47a49d2e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Level; import java.util.logging.Logger; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 184858ab747..16d0d823ac2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.yahoo.config.provision.NodeType; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index 19abe784749..c0b4ecb1a36 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.acl; import com.google.common.net.InetAddresses; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 4c5ad7c5c3a..3cb74aef5a7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.identity; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index e6d6f9463d3..ca2e7d192dd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.HostName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 1a79edca203..12628933e9a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -5,7 +5,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.flags.DoubleFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java index 0c03dc7f483..577ff53ebdf 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.process; import com.yahoo.jdisc.Timer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.nio.file.Files; import java.nio.file.Path; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java index dbe6b984211..4bab918c5a7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.process; import com.yahoo.jdisc.Timer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.File; import java.io.IOException; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java index 3d2caaa2ce9..66927e6a2b9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.process; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 44dd023677c..6983ff7576d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -5,7 +5,7 @@ import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java index d08811e9255..00856e6653c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java index 4d04409aaf0..220387bd157 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.InfraDeployer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 94cca64794d..e6b1823f830 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java index 8e1cd801a15..2e7706d5701 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.TransientException; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index b0b8f7d8d2c..7eccbe8366e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TransientException; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceInstance; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 1a238d5c3b6..845b4a462c0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java index 461b7346119..aef6cce2a46 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.provision.persistence; import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import javax.naming.NamingException; import javax.naming.directory.Attribute; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index fb83385920c..df3d2dedb4e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Provisioner; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.NodeRepository; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 1683f60bbbe..d48f02e18e8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.exception.LoadBalancerServiceException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 524b524f8c4..e8d8cd209f5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -5,7 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index e3b7c62cc0d..a74671d4e26 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -14,7 +14,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.FlagSource; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java index b1f205f69d8..1c5ae323d04 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.time.TimeBudget; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java index 9900c8de752..d79accc5b36 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.orchestrator.model; import com.google.common.util.concurrent.UncheckedTimeoutException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java index 58edbdc4218..0b439b81ef6 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.orchestrator.resources; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException; import com.yahoo.vespa.orchestrator.OrchestratorImpl; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index 054d3320215..5d4090fea30 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.orchestrator.resources; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.Host; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java index 6e857563f9b..11afb8a849c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.orchestrator.resources; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException; import com.yahoo.vespa.orchestrator.BatchInternalErrorException; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java index 99f1f00e7dc..d50eb1c7e26 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.orchestrator.status; import com.yahoo.config.provision.ApplicationId; import com.yahoo.jdisc.Timer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java index dfaaecbcf82..7632bcb8936 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.orchestrator.status; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.curator.Curator; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java index 246a70992c7..27eea7a685b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jaxrs.annotation.Component; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java index ddff782cdb1..fc025d013c4 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java @@ -6,7 +6,7 @@ import com.yahoo.exception.ExceptionUtils; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.test.ManualClock; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index 104f41c39ed..84aa7f5e72f 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.service.monitor.DuperModelListener; import java.util.ArrayList; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index cb1c6d2d95f..d64b2342eaf 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -10,7 +10,7 @@ import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.service.monitor.CriticalRegion; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java index 316b810c682..4b6a654a202 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.executor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.time.Duration; import java.util.Optional; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java index 1e2547cc762..dcaba9d6755 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.executor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.time.Duration; import java.util.concurrent.ConcurrentHashMap; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java index 19399c79b43..f13d2ee006a 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.service.health; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.util.EntityUtils; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index e7a8d33ea14..552cb09da14 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -5,7 +5,7 @@ import com.google.inject.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.jrt.slobrok.api.Mirror; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java index edb9e4820d1..1a82b0ff42f 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java @@ -14,7 +14,7 @@ import java.util.logging.Logger; import com.yahoo.collections.LazyMap; import com.yahoo.collections.LazySet; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * An aggregation of data which is only written to from a single thread. diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java index fce5026349e..1b96178465a 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java @@ -11,7 +11,7 @@ import com.yahoo.concurrent.ThreadLocalDirectory; import com.yahoo.concurrent.ThreadLocalDirectory.Updater; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.metrics.ManagerConfig; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * This is the coordinating class owning the executor and the top level objects diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java index f6762434275..0a085b0c9ab 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java @@ -5,7 +5,7 @@ import java.util.logging.Logger; import org.HdrHistogram.DoubleHistogram; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * A gauge or a counter or... who knows? The class for storing a metric when the diff --git a/statistics/src/main/java/com/yahoo/statistics/Value.java b/statistics/src/main/java/com/yahoo/statistics/Value.java index e9cacdebbd5..4260093fb20 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Value.java +++ b/statistics/src/main/java/com/yahoo/statistics/Value.java @@ -7,7 +7,7 @@ import java.util.logging.Logger; import com.yahoo.container.StatisticsConfig; import com.yahoo.container.StatisticsConfig.Values.Operations; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.event.Event; import com.yahoo.statistics.SampleSet.Sampling; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index 144ffaea5b4..a6498386b9a 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -10,7 +10,7 @@ import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.Pkcs10Csr; diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java index 90f7a76b356..d1e85adb503 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java @@ -9,7 +9,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 3b463e1af92..b45bd0f4e9f 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -27,7 +27,7 @@ import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Text; import com.yahoo.vespa.config.content.LoadTypeConfig; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index e6bc2211bea..cdd691b1ac7 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -8,7 +8,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Result; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index bcd78c790fe..502ddcddc9c 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -12,7 +12,7 @@ import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.shared.SharedSourceSession; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java index 752f81072e5..7fb9d019e6c 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -9,7 +9,7 @@ import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.documentapi.metrics.DocumentOperationStatus; import com.yahoo.documentapi.metrics.DocumentOperationType; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Trace; diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java index 581a51ffc02..42cff1ff4e7 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java @@ -10,7 +10,7 @@ import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.routing.Route; import com.yahoo.vespaclient.config.FeederConfig; diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java index 4fcbcf4d634..9b83e1d09cb 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java @@ -2,7 +2,7 @@ package com.yahoo.feedapi; import com.yahoo.concurrent.SystemTimer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Reply; diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java index d88de7d0fef..6f2ccf70b2c 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java @@ -13,7 +13,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentListEntry; import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage; import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage; import com.yahoo.documentapi.messagebus.protocol.MapVisitorMessage; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import java.io.IOException; diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java index 50f5c9d484c..509a588dac9 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java @@ -9,7 +9,7 @@ import com.yahoo.documentapi.VisitorDestinationSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.messagebus.network.Identity; import com.yahoo.net.HostName; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java index f9bece33a39..df1ab24901f 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import org.apache.curator.framework.CuratorFramework; diff --git a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java index b8a96d916f5..0916c90bb2d 100644 --- a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.zookeeper; import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyStoreUtils; -- cgit v1.2.3 From 25f832e613bb6087327c293b527d5445babb4319 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 13:12:46 +0200 Subject: LogLevel.DEBUG -> Level.FINE --- .../InstanceConfirmationResource.java | 2 +- .../InstanceRefreshResource.java | 2 +- .../instanceconfirmation/InstanceValidator.java | 6 +-- .../clustercontroller/core/ClusterStateView.java | 2 +- .../clustercontroller/core/FleetController.java | 10 ++-- .../core/MasterElectionHandler.java | 4 +- .../vespa/clustercontroller/core/NodeInfo.java | 14 +++--- .../clustercontroller/core/NodeStateGatherer.java | 30 ++++++------ .../clustercontroller/core/StateChangeHandler.java | 28 +++++------ .../core/SystemStateBroadcaster.java | 20 ++++---- .../core/database/DatabaseHandler.java | 56 +++++++++++----------- .../core/database/MasterDataGatherer.java | 14 +++--- .../core/database/ZooKeeperDatabase.java | 20 ++++---- .../restapiv2/requests/SetNodeStateRequest.java | 2 +- .../core/rpc/RPCCommunicator.java | 10 ++-- .../clustercontroller/core/rpc/RpcServer.java | 20 ++++---- .../core/status/statuspage/StatusPageServer.java | 8 ++-- .../vespa/clustercontroller/core/DummyVdsNode.java | 48 +++++++++---------- .../vespa/clustercontroller/core/FakeTimer.java | 2 +- .../core/FleetControllerTest.java | 4 +- .../config/application/OverrideProcessor.java | 2 +- .../application/provider/FilesApplicationFile.java | 8 ++-- .../provider/FilesApplicationPackage.java | 4 +- .../application/provider/SchemaValidators.java | 10 ++-- .../com/yahoo/config/model/ConfigModelRepo.java | 2 +- .../yahoo/config/model/builder/xml/XmlHelper.java | 2 +- .../model/producer/AbstractConfigProducer.java | 2 +- .../java/com/yahoo/vespa/model/VespaModel.java | 12 ++--- .../application/validation/RankSetupValidator.java | 2 +- .../vespa/model/builder/UserConfigBuilder.java | 2 +- .../model/builder/xml/dom/VespaDomBuilder.java | 2 +- .../vespa/config/proxy/ConfigProxyRpcServer.java | 12 ++--- .../com/yahoo/vespa/config/proxy/MemoryCache.java | 6 +-- .../config/proxy/MemoryCacheConfigClient.java | 4 +- .../com/yahoo/vespa/config/proxy/ProxyServer.java | 2 +- .../vespa/config/proxy/RpcConfigSourceClient.java | 20 ++++---- .../config/proxy/UpstreamConfigSubscriber.java | 2 +- .../filedistribution/CachedFilesMaintainer.java | 2 +- .../FileDistributionRpcServer.java | 6 +-- .../filedistribution/UrlDownloadRpcServer.java | 2 +- .../vespa/config/server/ApplicationRepository.java | 6 +-- .../vespa/config/server/ConfigServerBootstrap.java | 4 +- .../yahoo/vespa/config/server/ConfigServerDB.java | 4 +- .../config/server/SuperModelRequestHandler.java | 4 +- .../config/server/application/Application.java | 2 +- .../CompressedApplicationInputStream.java | 8 ++-- .../application/ConfigConvergenceChecker.java | 2 +- .../application/PermanentApplicationPackage.java | 2 +- .../server/application/TenantApplications.java | 2 +- .../vespa/config/server/deploy/Deployment.java | 4 +- .../server/filedistribution/FileDirectory.java | 10 ++-- .../filedistribution/FileDistributionImpl.java | 4 +- .../config/server/filedistribution/FileServer.java | 12 ++--- .../vespa/config/server/host/HostRegistry.java | 6 +-- .../config/server/http/HttpGetConfigHandler.java | 2 +- .../vespa/config/server/http/HttpHandler.java | 2 +- .../http/SessionContentStatusListResponse.java | 2 +- .../vespa/config/server/http/TesterClient.java | 2 +- .../vespa/config/server/http/v2/HostHandler.java | 10 ++-- .../server/http/v2/HttpGetConfigHandler.java | 2 +- .../server/metrics/ClusterMetricsRetriever.java | 2 +- .../modelfactory/ActivatedModelsBuilder.java | 2 +- .../config/server/modelfactory/ModelsBuilder.java | 4 +- .../server/modelfactory/PreparedModelsBuilder.java | 6 +-- .../vespa/config/server/monitoring/Metrics.java | 6 +-- .../config/server/rpc/DelayedConfigResponses.java | 6 +-- .../config/server/rpc/GetConfigProcessor.java | 4 +- .../yahoo/vespa/config/server/rpc/RpcServer.java | 18 +++---- .../rpc/security/MultiTenantRpcAuthorizer.java | 6 +-- .../config/server/session/LocalSessionRepo.java | 6 +-- .../server/session/LocalSessionStateWatcher.java | 2 +- .../vespa/config/server/session/RemoteSession.java | 10 ++-- .../config/server/session/RemoteSessionRepo.java | 8 ++-- .../server/session/RemoteSessionStateWatcher.java | 4 +- .../config/server/session/SessionFactoryImpl.java | 12 ++--- .../config/server/session/SessionPreparer.java | 4 +- .../server/session/SessionZooKeeperClient.java | 2 +- .../config/server/tenant/TenantRepository.java | 6 +-- .../config/server/tenant/TenantRequestHandler.java | 2 +- .../server/zookeeper/InitializedCounter.java | 2 +- .../config/server/zookeeper/ZKApplicationFile.java | 4 +- .../server/deploy/DeployHandlerLoggerTest.java | 2 +- .../config/server/http/SessionHandlerTest.java | 2 +- .../core/config/HandlersConfigurerDi.java | 2 +- .../container/jdisc/LoggingRequestHandler.java | 2 +- .../jdisc/ThreadedHttpRequestHandler.java | 4 +- .../com/yahoo/restapi/JacksonJsonResponse.java | 2 +- .../di/componentgraph/core/ComponentGraph.java | 2 +- .../container/jdisc/messagebus/SessionCache.java | 10 ++-- .../src/main/java/com/yahoo/search/Query.java | 2 +- .../search/grouping/vespa/GroupingExecutor.java | 4 +- .../com/yahoo/search/handler/SearchHandler.java | 6 +-- .../yahoo/search/query/rewrite/RewriterUtils.java | 4 +- .../search/searchers/InputCheckingSearcher.java | 2 +- .../yahoo/vespa/streamingvisitors/VdsVisitor.java | 10 ++-- .../controller/athenz/impl/AthenzFacade.java | 4 +- .../controller/deployment/DeploymentTrigger.java | 2 +- .../proxy/ConfigServerRestExecutorImpl.java | 4 +- .../controller/restapi/filter/SignatureFilter.java | 2 +- .../com/yahoo/docproc/SimpleDocumentProcessor.java | 8 ++-- .../docproc/jdisc/DocumentProcessingHandler.java | 4 +- .../docproc/jdisc/DocumentProcessingTask.java | 2 +- .../jdisc/messagebus/MbusRequestContext.java | 2 +- .../docproc/jdisc/messagebus/MessageFactory.java | 2 +- .../docproc/util/JoinerDocumentProcessor.java | 2 +- .../docproc/util/SplitterDocumentProcessor.java | 12 ++--- .../yahoo/docprocs/indexing/IndexingProcessor.java | 14 +++--- .../com/yahoo/docprocs/indexing/ScriptManager.java | 10 ++-- .../document/DocumentTypeManagerConfigurer.java | 4 +- .../java/com/yahoo/documentapi/ProgressToken.java | 8 ++-- .../com/yahoo/documentapi/VisitorIterator.java | 26 +++++----- .../MessageBusVisitorDestinationSession.java | 2 +- .../messagebus/MessageBusVisitorSession.java | 32 ++++++------- .../messagebus/protocol/StoragePolicy.java | 6 +-- .../fileacquirer/FileAcquirerImpl.java | 4 +- .../filedistribution/CompressedFileReference.java | 6 +-- .../vespa/filedistribution/FileDownloader.java | 2 +- .../yahoo/vespa/filedistribution/FileReceiver.java | 10 ++-- .../filedistribution/FileReferenceDownloader.java | 4 +- .../security/athenz/AthenzAuthorizationFilter.java | 8 ++-- .../base/JsonSecurityRequestFilterBase.java | 2 +- .../com/yahoo/messagebus/jdisc/MbusClient.java | 2 +- .../com/yahoo/messagebus/jdisc/MbusServer.java | 6 +-- .../shared/SharedDestinationSession.java | 2 +- .../shared/SharedIntermediateSession.java | 2 +- .../yahoo/messagebus/shared/SharedMessageBus.java | 2 +- .../messagebus/shared/SharedSourceSession.java | 2 +- .../com/yahoo/logserver/AbstractPluginLoader.java | 2 +- .../com/yahoo/logserver/BuiltinPluginLoader.java | 4 +- .../src/main/java/com/yahoo/logserver/Flusher.java | 2 +- .../java/com/yahoo/logserver/LogDispatcher.java | 4 +- .../handlers/archive/ArchiverHandler.java | 4 +- .../logserver/handlers/archive/LogWriter.java | 2 +- .../yahoo/messagebus/DynamicThrottlePolicy.java | 4 +- .../main/java/com/yahoo/messagebus/SendProxy.java | 2 +- .../yahoo/messagebus/network/rpc/RPCTarget.java | 8 ++-- .../application/ApplicationMetricsRetriever.java | 4 +- .../metricsproxy/service/ConfigSentinelClient.java | 4 +- .../metricsproxy/service/HttpMetricFetcher.java | 6 +-- .../service/RemoteHealthMetricFetcher.java | 2 +- .../vespa/metricsproxy/service/SystemPoller.java | 12 ++--- .../ai/vespa/metricsproxy/telegraf/Telegraf.java | 2 +- .../metricsproxy/telegraf/TelegrafRegistry.java | 4 +- .../task/util/process/ProcessStarterImpl.java | 2 +- .../provision/persistence/DnsNameResolver.java | 2 +- .../provision/provisioning/InfraDeployerImpl.java | 4 +- .../provisioning/LoadBalancerProvisioner.java | 2 +- .../provision/provisioning/NodePrioritizer.java | 4 +- .../provisioning/NodeRepositoryProvisioner.java | 4 +- .../vespa/orchestrator/model/StorageNodeImpl.java | 2 +- .../vespa/orchestrator/resources/HostResource.java | 22 ++++----- .../resources/HostSuspensionResource.java | 10 ++-- .../orchestrator/status/HostInfosServiceImpl.java | 2 +- .../vespa/orchestrator/status/ZkStatusService.java | 2 +- .../com/yahoo/vespa/service/duper/DuperModel.java | 6 +-- .../service/slobrok/SlobrokMonitorManagerImpl.java | 2 +- .../yahoo/vespa/http/server/ClientFeederV3.java | 2 +- .../yahoo/feedapi/MessagePropertyProcessor.java | 2 +- .../main/java/com/yahoo/feedapi/SharedSender.java | 4 +- .../java/com/yahoo/vespavisit/VdsVisitTarget.java | 2 +- .../vespa/curator/CuratorCompletionWaiter.java | 4 +- 161 files changed, 504 insertions(+), 504 deletions(-) diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java index 59616bec749..2828880ff67 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java @@ -32,7 +32,7 @@ public class InstanceConfirmationResource { @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public InstanceConfirmation confirmInstance(InstanceConfirmation instanceConfirmation) { - log.log(LogLevel.DEBUG, instanceConfirmation.toString()); + log.log(Level.FINE, instanceConfirmation.toString()); if (!instanceValidator.isValidInstance(instanceConfirmation)) { log.log(LogLevel.ERROR, "Invalid instance: " + instanceConfirmation); throw new ForbiddenException("Instance is invalid"); diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java index c62e5009c8f..a1eab136aa2 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java @@ -34,7 +34,7 @@ public class InstanceRefreshResource { @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public InstanceConfirmation confirmInstanceRefresh(InstanceConfirmation instanceConfirmation) { - log.log(LogLevel.DEBUG, instanceConfirmation.toString()); + log.log(Level.FINE, instanceConfirmation.toString()); if (!instanceValidator.isValidRefresh(instanceConfirmation)) { log.log(LogLevel.ERROR, "Invalid instance refresh: " + instanceConfirmation); throw new ForbiddenException("Instance is invalid"); diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index b1958bf5eed..b101f0bce67 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -85,11 +85,11 @@ public class InstanceValidator { return false; } - log.log(LogLevel.DEBUG, () -> String.format("Validating instance %s.", providerUniqueId)); + log.log(Level.FINE, () -> String.format("Validating instance %s.", providerUniqueId)); PublicKey publicKey = keyProvider.getPublicKey(signedIdentityDocument.signingKeyVersion()); if (signer.hasValidSignature(signedIdentityDocument, publicKey)) { - log.log(LogLevel.DEBUG, () -> String.format("Instance %s is valid.", providerUniqueId)); + log.log(Level.FINE, () -> String.format("Instance %s is valid.", providerUniqueId)); return true; } log.log(LogLevel.ERROR, () -> String.format("Instance %s has invalid signature.", providerUniqueId)); @@ -100,7 +100,7 @@ public class InstanceValidator { // We'll have to perform some validation on the instance id and other fields of the attribute map. // Separate between tenant and node certificate as well. public boolean isValidRefresh(InstanceConfirmation confirmation) { - log.log(LogLevel.DEBUG, () -> String.format("Accepting refresh for instance with identity '%s', provider '%s', instanceId '%s'.", + log.log(Level.FINE, () -> String.format("Accepting refresh for instance with identity '%s', provider '%s', instanceId '%s'.", new AthenzService(confirmation.domain, confirmation.service).getFullName(), confirmation.provider, confirmation.attributes.get(SAN_DNS_ATTRNAME))); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java index 0fcb5b65237..eefa7dfb322 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java @@ -102,7 +102,7 @@ public class ClusterStateView { // The distributor may be old (null), or the distributor may not have updated // to the latest state version just yet. We log here with fine, because it may // also be a symptom of something wrong. - log.log(LogLevel.DEBUG, "Current state version is " + currentStateVersion + + log.log(Level.FINE, "Current state version is " + currentStateVersion + ", while host info received from distributor " + node.getNodeIndex() + " is " + hostVersion); return; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 82fb4c825b9..1e66c397b88 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -548,7 +548,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd responseCode = StatusPageResponse.ResponseCode.INTERNAL_SERVER_ERROR; message = "Internal Server Error"; hiddenMessage = ExceptionUtils.getStackTraceAsString(e);; - log.log(LogLevel.DEBUG, "Unknown exception thrown for request " + httpRequest.getRequest() + + log.log(Level.FINE, "Unknown exception thrown for request " + httpRequest.getRequest() + ": " + hiddenMessage); } @@ -671,7 +671,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd // If there's a pending DB store we have not yet been able to store the // current state bundle to ZK and must therefore _not_ allow it to be published. if (database.hasPendingClusterStateMetaDataStore()) { - log.log(LogLevel.DEBUG, "Can't publish current cluster state as it has one or more pending ZooKeeper stores"); + log.log(Level.FINE, "Can't publish current cluster state as it has one or more pending ZooKeeper stores"); return false; } boolean sentAny = false; @@ -682,7 +682,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd && currentTime >= nextStateSendTime) { if (currentTime < firstAllowedStateBroadcast) { - log.log(LogLevel.DEBUG, "Not set to broadcast states just yet, but as we have gotten info from all nodes we can do so safely."); + log.log(Level.FINE, "Not set to broadcast states just yet, but as we have gotten info from all nodes we can do so safely."); // Reset timer to only see warning once. firstAllowedStateBroadcast = currentTime; } @@ -1037,7 +1037,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd + stateVersionTracker.getCurrentVersion() + " to be in line.", timer.getCurrentTimeInMillis())); long currentTime = timer.getCurrentTimeInMillis(); firstAllowedStateBroadcast = currentTime + options.minTimeBeforeFirstSystemStateBroadcast; - log.log(LogLevel.DEBUG, "At time " + currentTime + " we set first system state broadcast time to be " + log.log(Level.FINE, "At time " + currentTime + " we set first system state broadcast time to be " + options.minTimeBeforeFirstSystemStateBroadcast + " ms after at time " + firstAllowedStateBroadcast + "."); didWork = true; } @@ -1067,7 +1067,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd tick(); } } catch (InterruptedException e) { - log.log(LogLevel.DEBUG, "Event thread stopped by interrupt exception: " + e); + log.log(Level.FINE, "Event thread stopped by interrupt exception: " + e); } catch (Throwable t) { t.printStackTrace(); log.log(LogLevel.ERROR, "Fatal error killed fleet controller", t); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java index 205a6e13242..8299e77e01a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java @@ -34,7 +34,7 @@ public class MasterElectionHandler implements MasterInterface { this.nextInLineCount = Integer.MAX_VALUE; // Only a given set of nodes can ever become master if (index > (totalCount - 1) / 2) { - log.log(LogLevel.DEBUG, "Cluster controller " + index + ": We can never become master and will always stay a follower."); + log.log(Level.FINE, "Cluster controller " + index + ": We can never become master and will always stay a follower."); } // Tag current time as when we have not seen any other master. Make sure we're not taking over at once for master that is on the way down masterGoneFromZooKeeperTime = timer.getCurrentTimeInMillis(); @@ -179,7 +179,7 @@ public class MasterElectionHandler implements MasterInterface { if (nextInLineCount != ourPosition) { nextInLineCount = ourPosition; if (ourPosition > 0) { - log.log(LogLevel.DEBUG, "Cluster controller " + index + ": We are now " + getPosition(nextInLineCount) + " in queue to take over being master."); + log.log(Level.FINE, "Cluster controller " + index + ": We are now " + getPosition(nextInLineCount) + " in queue to take over being master."); } } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java index f98de8ca581..12cfe979511 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java @@ -167,7 +167,7 @@ abstract public class NodeInfo implements Comparable { } if (prematureCrashCount != count) { prematureCrashCount = count; - log.log(LogLevel.DEBUG, "Premature crash count on " + toString() + " set to " + count); + log.log(Level.FINE, "Premature crash count on " + toString() + " set to " + count); } } public int getPrematureCrashCount() { return prematureCrashCount; } @@ -309,17 +309,17 @@ abstract public class NodeInfo implements Comparable { if (state.getState().oneOf("dsm") && !reportedState.getState().oneOf("dsm")) { wentDownWithStartTime = reportedState.getStartTimestamp(); wentDownAtClusterState = getNewestSystemStateSent(); - log.log(LogLevel.DEBUG, "Setting going down timestamp of node " + node + " to " + wentDownWithStartTime); + log.log(Level.FINE, "Setting going down timestamp of node " + node + " to " + wentDownWithStartTime); } if (state.getState().equals(State.DOWN) && !reportedState.getState().oneOf("d")) { downStableStateTime = time; - log.log(LogLevel.DEBUG, "Down stable state on " + toString() + " altered to " + time); + log.log(Level.FINE, "Down stable state on " + toString() + " altered to " + time); if (reportedState.getState() == State.INITIALIZING) { recentlyObservedUnstableDuringInit = true; } } else if (state.getState().equals(State.UP) && !reportedState.getState().oneOf("u")) { upStableStateTime = time; - log.log(LogLevel.DEBUG, "Up stable state on " + toString() + " altered to " + time); + log.log(Level.FINE, "Up stable state on " + toString() + " altered to " + time); } if (!state.getState().validReportedNodeState(node.getType())) { throw new IllegalStateException("Trying to set illegal reported node state: " + state); @@ -388,7 +388,7 @@ abstract public class NodeInfo implements Comparable { downgradeToRpcVersion(RPCCommunicator.LEGACY_SET_SYSTEM_STATE2_RPC_VERSION, methodName, timer); return true; } else if (timer.getCurrentTimeInMillis() - 2000 < adjustedVersionTime) { - log.log(LogLevel.DEBUG, () -> "Node " + toString() + " does not support " + methodName + " call. Version already downgraded, so ignoring it."); + log.log(Level.FINE, () -> "Node " + toString() + " does not support " + methodName + " call. Version already downgraded, so ignoring it."); return true; } } @@ -397,7 +397,7 @@ abstract public class NodeInfo implements Comparable { } private void downgradeToRpcVersion(int newVersion, String methodName, Timer timer) { - log.log(LogLevel.DEBUG, () -> String.format("Node %s does not support %s call. Downgrading to version %d.", + log.log(Level.FINE, () -> String.format("Node %s does not support %s call. Downgrading to version %d.", toString(), methodName, newVersion)); version = newVersion; nextAttemptTime = 0; @@ -453,7 +453,7 @@ abstract public class NodeInfo implements Comparable { && (wentDownAtClusterState == null || wentDownAtClusterState.getVersion() < stateBundle.getVersion()) && !stateBundle.getBaselineClusterState().getNodeState(node).getState().oneOf("dsm")) { - log.log(LogLevel.DEBUG, () -> String.format("Clearing going down timestamp of node %s after " + + log.log(Level.FINE, () -> String.format("Clearing going down timestamp of node %s after " + "receiving ack of cluster state bundle %s", node, stateBundle)); wentDownWithStartTime = 0; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java index f1fe29cf0e3..7e7092eb67e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java @@ -64,12 +64,12 @@ public class NodeStateGatherer { if (info.getTimeForNextStateRequestAttempt() > currentTime) continue; // too early if (info.getRpcAddress() == null || info.isRpcAddressOutdated()) { // Cannot query state of node without RPC address - log.log(LogLevel.DEBUG, "Not sending getNodeState request to node " + info.getNode() + ": Not in slobrok"); + log.log(Level.FINE, "Not sending getNodeState request to node " + info.getNode() + ": Not in slobrok"); NodeState reportedState = info.getReportedState().clone(); if (( ! reportedState.getState().equals(State.DOWN) && currentTime - info.getRpcAddressOutdatedTimestamp() > maxSlobrokDisconnectGracePeriod) || reportedState.getState().equals(State.STOPPING)) // Don't wait for grace period if we expect node to be stopping { - log.log(LogLevel.DEBUG, "Setting reported state to DOWN " + log.log(Level.FINE, "Setting reported state to DOWN " + (reportedState.getState().equals(State.STOPPING) ? "as node completed stopping." : "as node has been out of slobrok longer than " + maxSlobrokDisconnectGracePeriod + ".")); @@ -102,7 +102,7 @@ public class NodeStateGatherer { NodeInfo info = req.getNodeInfo(); if (!info.isPendingGetNodeStateRequest(req)) { - log.log(LogLevel.DEBUG, "Ignoring getnodestate response from " + info.getNode() + log.log(Level.FINE, "Ignoring getnodestate response from " + info.getNode() + " as request replied to is not the most recent pending request."); continue; } @@ -118,10 +118,10 @@ public class NodeStateGatherer { listener.handleNewNodeState(info, newState.clone()); info.setReportedState(newState, currentTime); } else { - log.log(LogLevel.DEBUG, "Ignoring get node state error. Need to resend"); + log.log(Level.FINE, "Ignoring get node state error. Need to resend"); } } else { - log.log(LogLevel.DEBUG, "Ignoring getnodestate response from " + info.getNode() + " as it was aborted by client"); + log.log(Level.FINE, "Ignoring getnodestate response from " + info.getNode() + " as it was aborted by client"); } continue; @@ -159,7 +159,7 @@ public class NodeStateGatherer { if (info.getReportedState().getState().oneOf("ui")) { eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + "RPC timeout talking to node.", NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setDescription(msg); } else if (req.getReply().getReturnCode() == ErrorCode.CONNECTION) { @@ -174,7 +174,7 @@ public class NodeStateGatherer { if (info.getReportedState().getState().oneOf("ui")) { eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setState(State.DOWN); @@ -186,7 +186,7 @@ public class NodeStateGatherer { if (info.getReportedState().getState().oneOf("ui")) { eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setState(State.DOWN).setDescription(msg); } else if (msg.equals("Connection timed out")) { @@ -194,14 +194,14 @@ public class NodeStateGatherer { msg = "Connection error: Timeout"; eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); } else { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } } else { msg = "Connection error: " + reason; if (info.getReportedState().getState().oneOf("ui")) { eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setDescription(msg); } @@ -212,7 +212,7 @@ public class NodeStateGatherer { if (info.getReportedState().getState().oneOf("ui")) { eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setDescription(msg); } @@ -223,19 +223,19 @@ public class NodeStateGatherer { if (info.getReportedState().getState().oneOf("ui")) { eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setState(State.DOWN).setDescription(msg + ": get node state"); } else if (req.getReply().getReturnCode() == 75004) { String msg = "Node refused to answer RPC request and is likely stopping: " + req.getReply().getReturnMessage(); // The node is shutting down and is not accepting requests from anyone if (info.getReportedState().getState().equals(State.STOPPING)) { - log.log(LogLevel.DEBUG, "Failed to get node state from " + info + " because it is still shutting down."); + log.log(Level.FINE, "Failed to get node state from " + info + " because it is still shutting down."); } else { if (info.getReportedState().getState().oneOf("ui")) { eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } } newState.setState(State.STOPPING).setDescription(msg); @@ -244,7 +244,7 @@ public class NodeStateGatherer { if (info.getReportedState().getState().oneOf("ui")) { eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setState(State.DOWN).setDescription(msg); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index a8aa15d5959..e9005509c52 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -51,7 +51,7 @@ public class StateChangeHandler { final DatabaseHandler database, final DatabaseHandler.Context dbContext) throws InterruptedException { int startTimestampsReset = 0; - log.log(LogLevel.DEBUG, String.format("handleAllDistributorsInSync invoked for state version %d", currentState.getVersion())); + log.log(Level.FINE, String.format("handleAllDistributorsInSync invoked for state version %d", currentState.getVersion())); for (NodeType nodeType : NodeType.getTypes()) { for (ConfiguredNode configuredNode : nodes) { final Node node = new Node(nodeType, configuredNode.index()); @@ -60,19 +60,19 @@ public class StateChangeHandler { if (nodeInfo != null && nodeState != null) { if (nodeState.getStartTimestamp() > nodeInfo.getStartTimestamp()) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, String.format("Storing away new start timestamp for node %s (%d)", + log.log(Level.FINE, String.format("Storing away new start timestamp for node %s (%d)", node, nodeState.getStartTimestamp())); } nodeInfo.setStartTimestamp(nodeState.getStartTimestamp()); } if (nodeState.getStartTimestamp() > 0) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, String.format("Resetting timestamp in cluster state for node %s", node)); + log.log(Level.FINE, String.format("Resetting timestamp in cluster state for node %s", node)); } ++startTimestampsReset; } } else if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, node + ": " + + log.log(Level.FINE, node + ": " + (nodeInfo == null ? "null" : nodeInfo.getStartTimestamp()) + ", " + (nodeState == null ? "null" : nodeState.getStartTimestamp())); } @@ -85,7 +85,7 @@ public class StateChangeHandler { stateMayHaveChanged = true; database.saveStartTimestamps(dbContext); } else { - log.log(LogLevel.DEBUG, "Found no start timestamps to reset in cluster state."); + log.log(Level.FINE, "Found no start timestamps to reset in cluster state."); } } @@ -147,13 +147,13 @@ public class StateChangeHandler { if (reportedState.getMinUsedBits() != currentState.getMinUsedBits()) { final int oldCount = currentState.getMinUsedBits(); final int newCount = reportedState.getMinUsedBits(); - log.log(LogLevel.DEBUG, + log.log(Level.FINE, String.format("Altering node state to reflect that min distribution bit count has changed from %d to %d", oldCount, newCount)); eventLog.add(NodeEvent.forBaseline(node, String.format("Altered min distribution bit count from %d to %d", oldCount, newCount), NodeEvent.Type.CURRENT, currentTime), isMaster); } else if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, String.format("Not altering state of %s in cluster state because new state is too similar: %s", + log.log(Level.FINE, String.format("Not altering state of %s in cluster state because new state is too similar: %s", node, currentState.getTextualDifference(reportedState))); } @@ -181,12 +181,12 @@ public class StateChangeHandler { } if (node.getReportedState().getState().equals(State.STOPPING)) { - log.log(LogLevel.DEBUG, "Node " + node.getNode() + " is no longer in slobrok. Was in stopping state, so assuming it has shut down normally. Setting node down"); + log.log(Level.FINE, "Node " + node.getNode() + " is no longer in slobrok. Was in stopping state, so assuming it has shut down normally. Setting node down"); NodeState ns = node.getReportedState().clone(); ns.setState(State.DOWN); handleNewReportedNodeState(currentClusterState, node, ns.clone(), nodeListener); } else { - log.log(LogLevel.DEBUG, "Node " + node.getNode() + " no longer in slobrok was in state " + node.getReportedState() + ". Waiting to see if it reappears in slobrok"); + log.log(Level.FINE, "Node " + node.getNode() + " no longer in slobrok was in state " + node.getReportedState() + ". Waiting to see if it reappears in slobrok"); } stateMayHaveChanged = true; @@ -209,7 +209,7 @@ public class StateChangeHandler { stateMayHaveChanged = true; if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, String.format("Got new wanted nodestate for %s: %s", node, currentState.getTextualDifference(proposedState))); + log.log(Level.FINE, String.format("Got new wanted nodestate for %s: %s", node, currentState.getTextualDifference(proposedState))); } // Should be checked earlier before state was set in cluster assert(proposedState.getState().validWantedNodeState(node.getNode().getType())); @@ -321,11 +321,11 @@ public class StateChangeHandler { if (mayResetCrashCounterOnStableUpNode(currentTime, node, lastReportedState)) { node.setPrematureCrashCount(0); - log.log(LogLevel.DEBUG, "Resetting premature crash count on node " + node + " as it has been up for a long time."); + log.log(Level.FINE, "Resetting premature crash count on node " + node + " as it has been up for a long time."); triggeredAnyTimers = true; } else if (mayResetCrashCounterOnStableDownNode(currentTime, node, lastReportedState)) { node.setPrematureCrashCount(0); - log.log(LogLevel.DEBUG, "Resetting premature crash count on node " + node + " as it has been down for a long time."); + log.log(Level.FINE, "Resetting premature crash count on node " + node + " as it has been down for a long time."); triggeredAnyTimers = true; } @@ -416,7 +416,7 @@ public class StateChangeHandler { final NodeStateOrHostInfoChangeHandler nodeListener) { final long timeNow = timer.getCurrentTimeInMillis(); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, String.format("Finding new cluster state entry for %s switching state %s", + log.log(Level.FINE, String.format("Finding new cluster state entry for %s switching state %s", node, currentState.getTextualDifference(reportedState))); } @@ -482,7 +482,7 @@ public class StateChangeHandler { if (nodeUpToDownEdge(node, currentState, reportedState)) { node.setTransitionTime(timeNow); if (node.getUpStableStateTime() + stableStateTimePeriod > timeNow && !isControlledShutdown(reportedState)) { - log.log(LogLevel.DEBUG, "Stable state: " + node.getUpStableStateTime() + " + " + stableStateTimePeriod + " > " + timeNow); + log.log(Level.FINE, "Stable state: " + node.getUpStableStateTime() + " + " + stableStateTimePeriod + " > " + timeNow); eventLog.add(NodeEvent.forBaseline(node, String.format("Stopped or possibly crashed after %d ms, which is before " + "stable state time period. Premature crash count is now %d.", diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java index 55fc0294dd7..d6b8364954b 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java @@ -92,11 +92,11 @@ public class SystemStateBroadcaster { // NO_SUCH_METHOD implies node is on a version that does not understand explicit activations // and it has already merrily started using the state version. Treat as if it had been ACKed. if (reply.getReturnCode() != ErrorCode.NO_SUCH_METHOD) { - log.log(LogLevel.DEBUG, () -> String.format("Activation NACK for node %s with version %d, message %s", + log.log(Level.FINE, () -> String.format("Activation NACK for node %s with version %d, message %s", info, version, reply.getReturnMessage())); success = false; } else { - log.log(LogLevel.DEBUG, () -> String.format("Node %s did not understand state activation RPC; " + + log.log(Level.FINE, () -> String.format("Node %s did not understand state activation RPC; " + "implicitly treating state %d as activated on node", info, version)); } } else if (reply.getActualVersion() != version) { @@ -109,7 +109,7 @@ public class SystemStateBroadcaster { version, info, reply.getActualVersion())); success = false; } else { - log.log(LogLevel.DEBUG, () -> String.format("Node %s reports successful activation of state " + + log.log(Level.FINE, () -> String.format("Node %s reports successful activation of state " + "version %d", info, version)); } info.setSystemStateVersionActivationAcked(version, success); @@ -140,7 +140,7 @@ public class SystemStateBroadcaster { } } else { info.setClusterStateBundleVersionAcknowledged(version, true); - log.log(LogLevel.DEBUG, () -> String.format("Node %s ACKed system state version %d.", info, version)); + log.log(Level.FINE, () -> String.format("Node %s ACKed system state version %d.", info, version)); lastErrorReported.remove(info.getNode()); } } @@ -216,7 +216,7 @@ public class SystemStateBroadcaster { if (!anyDistributorsNeedStateBundle && (currentStateVersion > lastStateVersionBundleAcked)) { markCurrentClusterStateBundleAsReceivedByAllDistributors(); if (clusterStateBundle.deferredActivation()) { - log.log(LogLevel.DEBUG, () -> String.format("All distributors have ACKed cluster state " + + log.log(Level.FINE, () -> String.format("All distributors have ACKed cluster state " + "version %d, sending activation", currentStateVersion)); } else { markCurrentClusterStateAsConverged(database, dbContext, fleetController); @@ -235,7 +235,7 @@ public class SystemStateBroadcaster { if (!anyDistributorsNeedActivation && (currentStateVersion > lastClusterStateVersionConverged)) { markCurrentClusterStateAsConverged(database, dbContext, fleetController); } else { - log.log(LogLevel.DEBUG, () -> String.format("distributors still need activation in state %d (last converged: %d)", + log.log(Level.FINE, () -> String.format("distributors still need activation in state %d (last converged: %d)", currentStateVersion, lastClusterStateVersionConverged)); } } @@ -245,7 +245,7 @@ public class SystemStateBroadcaster { } private void markCurrentClusterStateAsConverged(DatabaseHandler database, DatabaseHandler.Context dbContext, FleetController fleetController) throws InterruptedException { - log.log(LogLevel.DEBUG, "All distributors have newest clusterstate, updating start timestamps in zookeeper and clearing them from cluster state"); + log.log(Level.FINE, "All distributors have newest clusterstate, updating start timestamps in zookeeper and clearing them from cluster state"); lastClusterStateVersionConverged = clusterStateBundle.getVersion(); lastClusterStateBundleConverged = clusterStateBundle; fleetController.handleAllDistributorsInSync(database, dbContext); @@ -272,11 +272,11 @@ public class SystemStateBroadcaster { if (nodeNeedsToObserveStartupTimestamps(node)) { // TODO this is the same for all nodes, compute only once ClusterStateBundle modifiedBundle = clusterStateBundle.cloneWithMapper(state -> buildModifiedClusterState(state, dbContext)); - log.log(LogLevel.DEBUG, () -> String.format("Sending modified cluster state version %d" + + log.log(Level.FINE, () -> String.format("Sending modified cluster state version %d" + " to node %s: %s", baselineState.getVersion(), node, modifiedBundle)); communicator.setSystemState(modifiedBundle, node, setClusterStateWaiter); } else { - log.log(LogLevel.DEBUG, () -> String.format("Sending system state version %d to node %s. " + + log.log(Level.FINE, () -> String.format("Sending system state version %d to node %s. " + "(went down time %d, node start time %d)", baselineState.getVersion(), node, node.getWentDownWithStartTime(), node.getStartTimestamp())); communicator.setSystemState(clusterStateBundle, node, setClusterStateWaiter); @@ -297,7 +297,7 @@ public class SystemStateBroadcaster { var recipients = resolveStateActivationSendSet(dbContext); for (NodeInfo node : recipients) { - log.log(LogLevel.DEBUG, () -> String.format("Sending cluster state activation to node %s for version %d", + log.log(Level.FINE, () -> String.format("Sending cluster state activation to node %s for version %d", node, clusterStateBundle.getVersion())); communicator.activateClusterStateVersion(clusterStateBundle.getVersion(), node, activateClusterStateVersionWaiter); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java index 78e71d63e6a..a7570ce5028 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java @@ -52,7 +52,7 @@ public class DatabaseHandler { } private class DatabaseListener implements Database.DatabaseListener { public void handleZooKeeperSessionDown() { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Lost contact with zookeeper server"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Lost contact with zookeeper server"); synchronized(monitor) { lostZooKeeperConnectionEvent = true; monitor.notifyAll(); @@ -62,7 +62,7 @@ public class DatabaseHandler { public void handleMasterData(Map data) { synchronized (monitor) { if (masterDataEvent != null && masterDataEvent.equals(data)) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": New master data was the same as the last one. Not responding to it"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": New master data was the same as the last one. Not responding to it"); } else { masterDataEvent = data; } @@ -133,7 +133,7 @@ public class DatabaseHandler { currentlyStored.clear(); pendingStore.clear(); pendingStore.masterVote = currentVote; - log.log(LogLevel.DEBUG, "Cleared session metadata. Pending master vote is now " + log.log(Level.FINE, "Cleared session metadata. Pending master vote is now " + pendingStore.masterVote); } @@ -149,7 +149,7 @@ public class DatabaseHandler { public void setZooKeeperSessionTimeout(int timeout) { if (timeout == zooKeeperSessionTimeout) return; - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Got new ZooKeeper session timeout of " + timeout + " milliseconds."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Got new ZooKeeper session timeout of " + timeout + " milliseconds."); zooKeeperSessionTimeout = timeout; reset(); } @@ -176,7 +176,7 @@ public class DatabaseHandler { database = databaseFactory.create(params); } } catch (KeeperException.NodeExistsException e) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Cannot create ephemeral fleetcontroller node. ZooKeeper server " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Cannot create ephemeral fleetcontroller node. ZooKeeper server " + "not seen old fleetcontroller instance disappear? It already exists. Will retry later: " + e.getMessage()); } catch (InterruptedException e) { throw (InterruptedException) new InterruptedException("Interrupted").initCause(e); @@ -202,17 +202,17 @@ public class DatabaseHandler { synchronized (monitor) { if (zooKeeperAddress == null) return false; // If not using zookeeper no work to be done if (lostZooKeeperConnectionEvent) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": doNextZooKeeperTask(): lost connection"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": doNextZooKeeperTask(): lost connection"); context.getFleetController().lostDatabaseConnection(); lostZooKeeperConnectionEvent = false; didWork = true; if (masterDataEvent != null) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Had new master data queued on disconnect. Removing master data event"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Had new master data queued on disconnect. Removing master data event"); masterDataEvent = null; } } if (masterDataEvent != null) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": doNextZooKeeperTask(): new master data"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": doNextZooKeeperTask(): new master data"); if (!masterDataEvent.containsKey(nodeIndex)) { Integer currentVote = (pendingStore.masterVote != null ? pendingStore.masterVote : currentlyStored.masterVote); assert(currentVote != null); @@ -257,10 +257,10 @@ public class DatabaseHandler { boolean didWork = false; if (pendingStore.masterVote != null) { didWork = true; - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Attempting to store master vote " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Attempting to store master vote " + pendingStore.masterVote + " into zookeeper."); if (database.storeMasterVote(pendingStore.masterVote)) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Managed to store master vote " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Managed to store master vote " + pendingStore.masterVote + " into zookeeper."); currentlyStored.masterVote = pendingStore.masterVote; pendingStore.masterVote = null; @@ -271,7 +271,7 @@ public class DatabaseHandler { } if (pendingStore.lastSystemStateVersion != null) { didWork = true; - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Attempting to store last system state version " + pendingStore.lastSystemStateVersion + " into zookeeper."); if (database.storeLatestSystemStateVersion(pendingStore.lastSystemStateVersion)) { @@ -283,7 +283,7 @@ public class DatabaseHandler { } if (pendingStore.startTimestamps != null) { didWork = true; - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Attempting to store " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Attempting to store " + pendingStore.startTimestamps.size() + " start timestamps into zookeeper."); if (database.storeStartTimestamps(pendingStore.startTimestamps)) { currentlyStored.startTimestamps = pendingStore.startTimestamps; @@ -294,7 +294,7 @@ public class DatabaseHandler { } if (pendingStore.wantedStates != null) { didWork = true; - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Attempting to store " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Attempting to store " + pendingStore.wantedStates.size() + " wanted states into zookeeper."); if (database.storeWantedStates(pendingStore.wantedStates)) { currentlyStored.wantedStates = pendingStore.wantedStates; @@ -316,7 +316,7 @@ public class DatabaseHandler { } public void setMasterVote(Context context, int wantedMasterCandidate) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Checking if master vote has been updated and need to be stored."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Checking if master vote has been updated and need to be stored."); // Schedule a write if one of the following is true: // - There is already a pending vote to be written, that may have been written already without our knowledge // - We don't know what is actually stored now @@ -324,14 +324,14 @@ public class DatabaseHandler { if (pendingStore.masterVote != null || currentlyStored.masterVote == null || currentlyStored.masterVote != wantedMasterCandidate) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Scheduling master vote " + wantedMasterCandidate + " to be stored in zookeeper."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Scheduling master vote " + wantedMasterCandidate + " to be stored in zookeeper."); pendingStore.masterVote = wantedMasterCandidate; doNextZooKeeperTask(context); } } public void saveLatestSystemStateVersion(Context context, int version) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Checking if latest system state version has been updated and need to be stored."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Checking if latest system state version has been updated and need to be stored."); // Schedule a write if one of the following is true: // - There is already a pending vote to be written, that may have been written already without our knowledge // - We don't know what is actually stored now @@ -339,14 +339,14 @@ public class DatabaseHandler { if (pendingStore.lastSystemStateVersion != null || currentlyStored.lastSystemStateVersion == null || currentlyStored.lastSystemStateVersion != version) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Scheduling new last system state version " + version + " to be stored in zookeeper."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Scheduling new last system state version " + version + " to be stored in zookeeper."); pendingStore.lastSystemStateVersion = version; doNextZooKeeperTask(context); } } public int getLatestSystemStateVersion() throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Retrieving latest system state version."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Retrieving latest system state version."); synchronized (databaseMonitor) { if (database != null && !database.isClosed()) { currentlyStored.lastSystemStateVersion = database.retrieveLatestSystemStateVersion(); @@ -363,7 +363,7 @@ public class DatabaseHandler { } public void saveLatestClusterStateBundle(Context context, ClusterStateBundle clusterStateBundle) throws InterruptedException { - log.log(LogLevel.DEBUG, () -> String.format("Fleetcontroller %d: Scheduling bundle %s to be saved to ZooKeeper", nodeIndex, clusterStateBundle)); + log.log(Level.FINE, () -> String.format("Fleetcontroller %d: Scheduling bundle %s to be saved to ZooKeeper", nodeIndex, clusterStateBundle)); pendingStore.clusterStateBundle = clusterStateBundle; doNextZooKeeperTask(context); } @@ -378,7 +378,7 @@ public class DatabaseHandler { } public ClusterStateBundle getLatestClusterStateBundle() throws InterruptedException { - log.log(LogLevel.DEBUG, () -> String.format("Fleetcontroller %d: Retrieving latest cluster state bundle from ZooKeeper", nodeIndex)); + log.log(Level.FINE, () -> String.format("Fleetcontroller %d: Retrieving latest cluster state bundle from ZooKeeper", nodeIndex)); synchronized (databaseMonitor) { if (database != null && !database.isClosed()) { return database.retrieveLastPublishedStateBundle(); @@ -389,7 +389,7 @@ public class DatabaseHandler { } public void saveWantedStates(Context context) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Checking whether wanted states have changed compared to zookeeper version."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Checking whether wanted states have changed compared to zookeeper version."); Map wantedStates = new TreeMap<>(); for (NodeInfo info : context.getCluster().getNodeInfo()) { if (!info.getUserWantedState().equals(new NodeState(info.getNode().getType(), State.UP))) { @@ -403,14 +403,14 @@ public class DatabaseHandler { if (pendingStore.wantedStates != null || currentlyStored.wantedStates == null || !currentlyStored.wantedStates.equals(wantedStates)) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Scheduling new wanted states to be stored into zookeeper."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Scheduling new wanted states to be stored into zookeeper."); pendingStore.wantedStates = wantedStates; doNextZooKeeperTask(context); } } public boolean loadWantedStates(Context context) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Retrieving node wanted states."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Retrieving node wanted states."); synchronized (databaseMonitor) { if (database != null && !database.isClosed()) { currentlyStored.wantedStates = database.retrieveWantedStates(); @@ -421,7 +421,7 @@ public class DatabaseHandler { if (usingZooKeeper()) { // We get here if the ZooKeeper client has lost the connection to ZooKeeper. // TODO: Should instead fail the tick until connected!? - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Failed to retrieve wanted states from ZooKeeper. Assuming UP for all nodes."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Failed to retrieve wanted states from ZooKeeper. Assuming UP for all nodes."); } wantedStates = new TreeMap<>(); } @@ -435,7 +435,7 @@ public class DatabaseHandler { context.getNodeStateUpdateListener().handleNewWantedNodeState(nodeInfo, wantedState); altered = true; } - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Node " + node + " has wanted state " + wantedState); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Node " + node + " has wanted state " + wantedState); } // Remove wanted state from any node having a wanted state set that is no longer valid @@ -451,13 +451,13 @@ public class DatabaseHandler { } public void saveStartTimestamps(Context context) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Scheduling start timestamps to be stored into zookeeper."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Scheduling start timestamps to be stored into zookeeper."); pendingStore.startTimestamps = context.getCluster().getStartTimestamps(); doNextZooKeeperTask(context); } public boolean loadStartTimestamps(ContentCluster cluster) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Retrieving start timestamps"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Retrieving start timestamps"); synchronized (databaseMonitor) { if (database == null || database.isClosed()) { return false; @@ -473,7 +473,7 @@ public class DatabaseHandler { } for (Map.Entry e : startTimestamps.entrySet()) { cluster.setStartTimestamp(e.getKey(), e.getValue()); - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Node " + e.getKey() + " has start timestamp " + e.getValue()); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Node " + e.getKey() + " has start timestamp " + e.getValue()); } return true; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java index 0fdfd46dec5..28527162b71 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java @@ -68,10 +68,10 @@ public class MasterDataGatherer { case NodeDeleted: // We get this event when fleetcontrollers shut down and node in dir disappears. But it should also trigger a NodeChildrenChanged event, so // ignoring this one. - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Node deleted event gotten. Ignoring it, expecting a NodeChildrenChanged event too."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Node deleted event gotten. Ignoring it, expecting a NodeChildrenChanged event too."); break; case None: - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Got ZooKeeper event None."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Got ZooKeeper event None."); } } } @@ -90,7 +90,7 @@ public class MasterDataGatherer { for (String node : nodes) { int index = Integer.parseInt(node); nextMasterData.put(index, null); - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Attempting to fetch data in node '" + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Attempting to fetch data in node '" + zooKeeperRoot + index + "' to see vote"); session.getData(zooKeeperRoot + "indexes/" + index, changeWatcher, nodeListener, null); // Invocation of cycleCompleted() for fully accumulated election state will happen @@ -127,7 +127,7 @@ public class MasterDataGatherer { Integer value = Integer.valueOf(data); if (nextMasterData.containsKey(index)) { if (value.equals(nextMasterData.get(index))) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + ", which already was " + value + "."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + ", which already was " + value + "."); } else { log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + ". Altering vote from " + nextMasterData.get(index) + " to " + value + "."); nextMasterData.put(index, value); @@ -143,7 +143,7 @@ public class MasterDataGatherer { } } } - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Got votes for all fleetcontrollers. Sending event with new fleet data for update"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Got votes for all fleetcontrollers. Sending event with new fleet data for update"); cycleCompleted(); } } @@ -173,14 +173,14 @@ public class MasterDataGatherer { Map copy; synchronized (nextMasterData) { if (nextMasterData.equals(masterData)) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": No change in master data detected, not sending it on"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": No change in master data detected, not sending it on"); // for(Integer i : nextMasterData.keySet()) { System.err.println(i + " -> " + nextMasterData.get(i)); } return; } masterData = new TreeMap(nextMasterData); copy = masterData; } - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Got new master data, sending it on"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Got new master data, sending it on"); listener.handleMasterData(copy); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java index 453216039d9..a2d370a6fe8 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java @@ -114,13 +114,13 @@ public class ZooKeeperDatabase extends Database { private void createNode(String prefix, String nodename, byte value[]) throws KeeperException, InterruptedException { try{ if (session.exists(prefix + nodename, false) != null) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Zookeeper node '" + prefix + nodename + "' already exists. Not creating it"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Zookeeper node '" + prefix + nodename + "' already exists. Not creating it"); return; } session.create(prefix + nodename, value, acl, CreateMode.PERSISTENT); - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Created zookeeper node '" + prefix + nodename + "'"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Created zookeeper node '" + prefix + nodename + "'"); } catch (KeeperException.NodeExistsException e) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Node to create existed, " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Node to create existed, " + "but this is normal as other nodes may create them at the same time."); } } @@ -162,7 +162,7 @@ public class ZooKeeperDatabase extends Database { public void close() { sessionOpen = false; try{ - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Trying to close ZooKeeper session 0x" + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Trying to close ZooKeeper session 0x" + Long.toHexString(session.getSessionId())); session.close(); } catch (InterruptedException e) { @@ -217,7 +217,7 @@ public class ZooKeeperDatabase extends Database { public Integer retrieveLatestSystemStateVersion() throws InterruptedException { Stat stat = new Stat(); try{ - log.log(LogLevel.DEBUG, () -> String.format("Fleetcontroller %d: Fetching latest cluster state at '%slatestversion'", + log.log(Level.FINE, () -> String.format("Fleetcontroller %d: Fetching latest cluster state at '%slatestversion'", nodeIndex, zooKeeperRoot)); byte[] data = session.getData(zooKeeperRoot + "latestversion", false, stat); lastKnownStateVersionZNodeVersion = stat.getVersion(); @@ -249,7 +249,7 @@ public class ZooKeeperDatabase extends Database { } byte val[] = sb.toString().getBytes(utf8); try{ - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Storing wanted states at '" + zooKeeperRoot + "wantedstates'"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Storing wanted states at '" + zooKeeperRoot + "wantedstates'"); session.setData(zooKeeperRoot + "wantedstates", val, -1); return true; } catch (InterruptedException e) { @@ -262,7 +262,7 @@ public class ZooKeeperDatabase extends Database { public Map retrieveWantedStates() throws InterruptedException { try{ - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Fetching wanted states at '" + zooKeeperRoot + "wantedstates'"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Fetching wanted states at '" + zooKeeperRoot + "wantedstates'"); Stat stat = new Stat(); byte[] data = session.getData(zooKeeperRoot + "wantedstates", false, stat); Map wanted = new TreeMap<>(); @@ -300,7 +300,7 @@ public class ZooKeeperDatabase extends Database { } byte val[] = sb.toString().getBytes(utf8); try{ - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Storing start timestamps at '" + zooKeeperRoot + "starttimestamps"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Storing start timestamps at '" + zooKeeperRoot + "starttimestamps"); session.setData(zooKeeperRoot + "starttimestamps", val, -1); return true; } catch (InterruptedException e) { @@ -314,7 +314,7 @@ public class ZooKeeperDatabase extends Database { @Override public Map retrieveStartTimestamps() throws InterruptedException { try{ - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Fetching start timestamps at '" + zooKeeperRoot + "starttimestamps'"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Fetching start timestamps at '" + zooKeeperRoot + "starttimestamps'"); Stat stat = new Stat(); byte[] data = session.getData(zooKeeperRoot + "starttimestamps", false, stat); Map wanted = new TreeMap(); @@ -347,7 +347,7 @@ public class ZooKeeperDatabase extends Database { EnvelopedClusterStateBundleCodec envelopedBundleCodec = new SlimeClusterStateBundleCodec(); byte[] encodedBundle = envelopedBundleCodec.encodeWithEnvelope(stateBundle); try{ - log.log(LogLevel.DEBUG, () -> String.format("Fleetcontroller %d: Storing published state bundle %s at " + + log.log(Level.FINE, () -> String.format("Fleetcontroller %d: Storing published state bundle %s at " + "'%spublished_state_bundle' with expected znode version %d", nodeIndex, stateBundle, zooKeeperRoot, lastKnownStateBundleZNodeVersion)); var stat = session.setData(zooKeeperRoot + "published_state_bundle", encodedBundle, lastKnownStateBundleZNodeVersion); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java index 64ca08dc997..94b1a9e1fbc 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java @@ -117,7 +117,7 @@ public class SetNodeStateRequest extends Request { NodeStateChangeChecker.Result result = cluster.calculateEffectOfNewState( node, currentClusterState, condition, wantedState, newWantedState); - log.log(LogLevel.DEBUG, "node=" + node + + log.log(Level.FINE, "node=" + node + " current-cluster-state=" + currentClusterState + // Includes version in output format " condition=" + condition + " wanted-state=" + wantedState + diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java index a2619692961..d8a72a9272c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java @@ -110,7 +110,7 @@ public class RPCCommunicator implements Communicator { public void getNodeState(NodeInfo node, Waiter externalWaiter) { Target connection = getConnection(node); if ( ! connection.isValid()) { - log.log(LogLevel.DEBUG, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); + log.log(Level.FINE, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); } NodeState currentState = node.getReportedState(); Request req = new Request("getnodestate3"); @@ -138,7 +138,7 @@ public class RPCCommunicator implements Communicator { Target connection = getConnection(node); if ( ! connection.isValid()) { - log.log(LogLevel.DEBUG, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); + log.log(Level.FINE, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); return; } int nodeVersion = node.getVersion(); @@ -156,7 +156,7 @@ public class RPCCommunicator implements Communicator { v.add(new DataValue(encodedBundle.getCompression().data())); } - log.log(LogLevel.DEBUG, () -> String.format("Sending '%s' RPC to %s for state version %d", + log.log(Level.FINE, () -> String.format("Sending '%s' RPC to %s for state version %d", req.methodName(), node.getRpcAddress(), stateBundle.getVersion())); RPCSetClusterStateRequest stateRequest = new RPCSetClusterStateRequest(node, req, baselineState.getVersion()); waiter.setRequest(stateRequest); @@ -171,14 +171,14 @@ public class RPCCommunicator implements Communicator { Target connection = getConnection(node); if ( ! connection.isValid()) { - log.log(LogLevel.DEBUG, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); + log.log(Level.FINE, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); return; } var req = new Request(ACTIVATE_CLUSTER_STATE_VERSION_RPC_METHOD_NAME); req.parameters().add(new Int32Value(clusterStateVersion)); - log.log(LogLevel.DEBUG, () -> String.format("Sending '%s' RPC to %s for state version %d", + log.log(Level.FINE, () -> String.format("Sending '%s' RPC to %s for state version %d", req.methodName(), node.getRpcAddress(), clusterStateVersion)); var activationRequest = new RPCActivateClusterStateVersionRequest(node, req, clusterStateVersion); waiter.setRequest(activationRequest); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index d368d377aee..e3ae95b0abe 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -97,13 +97,13 @@ public class RpcServer { public void connect() throws ListenFailedException, UnknownHostException { disconnect(); - log.log(LogLevel.DEBUG, "Fleetcontroller " + fleetControllerIndex + ": Connecting RPC server."); + log.log(Level.FINE, "Fleetcontroller " + fleetControllerIndex + ": Connecting RPC server."); if (supervisor != null) disconnect(); supervisor = new Supervisor(new Transport()); addMethods(); - log.log(LogLevel.DEBUG, "Fleetcontroller " + fleetControllerIndex + ": Attempting to bind to port " + port); + log.log(Level.FINE, "Fleetcontroller " + fleetControllerIndex + ": Attempting to bind to port " + port); acceptor = supervisor.listen(new Spec(port)); - log.log(LogLevel.DEBUG, "Fleetcontroller " + fleetControllerIndex + ": RPC server listening to port " + acceptor.port()); + log.log(Level.FINE, "Fleetcontroller " + fleetControllerIndex + ": RPC server listening to port " + acceptor.port()); StringBuffer slobroks = new StringBuffer("("); for (String s : slobrokConnectionSpecs) { slobroks.append(" ").append(s); @@ -123,7 +123,7 @@ public class RpcServer { public void disconnect() { if (register != null) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + fleetControllerIndex + ": Disconnecting RPC server."); + log.log(Level.FINE, "Fleetcontroller " + fleetControllerIndex + ": Disconnecting RPC server."); register.shutdown(); register = null; } @@ -218,7 +218,7 @@ public class RpcServer { } try{ if (req.methodName().equals("getMaster")) { - log.log(LogLevel.DEBUG, "Resolving RPC getMaster request"); + log.log(Level.FINE, "Resolving RPC getMaster request"); Integer master = masterHandler.getMaster(); String masterReason = masterHandler.getMasterReason(); req.returnValues().add(new Int32Value(master == null ? -1 : master)); @@ -230,7 +230,7 @@ public class RpcServer { throw new IllegalStateException("Refusing to answer RPC calls as we are not the master fleetcontroller."); } if (req.methodName().equals("getNodeList")) { - log.log(LogLevel.DEBUG, "Resolving RPC getNodeList request"); + log.log(Level.FINE, "Resolving RPC getNodeList request"); List slobrok = new ArrayList(); List rpc = new ArrayList(); for(NodeInfo node : cluster.getNodeInfo()) { @@ -244,12 +244,12 @@ public class RpcServer { req.returnValues().add(new StringArray(rpc.toArray(new String[rpc.size()]))); req.returnRequest(); } else if (req.methodName().equals("getSystemState")) { - log.log(LogLevel.DEBUG, "Resolving RPC getSystemState request"); + log.log(Level.FINE, "Resolving RPC getSystemState request"); req.returnValues().add(new StringValue("")); req.returnValues().add(new StringValue(systemState.toString(true))); req.returnRequest(); } else if (req.methodName().equals("getNodeState")) { - log.log(LogLevel.DEBUG, "Resolving RPC getNodeState request"); + log.log(Level.FINE, "Resolving RPC getNodeState request"); NodeType nodeType = NodeType.get(req.parameters().get(0).asString()); int nodeIndex = req.parameters().get(1).asInt32(); @@ -296,7 +296,7 @@ public class RpcServer { changeListener.handleNewWantedNodeState(node, nodeState); } else { message = "Node " + node + " already had wanted state " + nodeState.toString(); - log.log(LogLevel.DEBUG, message); + log.log(Level.FINE, message); } req.returnValues().add(new StringValue(message)); req.returnRequest(); @@ -309,7 +309,7 @@ public class RpcServer { if (log.isLoggable(LogLevel.DEBUG)) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); - log.log(LogLevel.DEBUG, "Failed RPC Request: " + sw); + log.log(Level.FINE, "Failed RPC Request: " + sw); } String errorMsg = e.getMessage(); if (errorMsg == null) { errorMsg = e.toString(); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index c7f7682a62a..8e6cbc28a85 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -85,7 +85,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { if (!isConnected()) { log.log(LogLevel.INFO, "Fleetcontroller: Server Socket not ready after connect()"); } - log.log(LogLevel.DEBUG, "Fleet controller status page viewer listening to " + ssocket.getLocalSocketAddress()); + log.log(Level.FINE, "Fleet controller status page viewer listening to " + ssocket.getLocalSocketAddress()); monitor.notifyAll(); } } @@ -149,7 +149,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { log.log(shouldBeConnected ? LogLevel.WARNING : LogLevel.DEBUG, "Caught IO exception in ServerSocket.accept(): " + e.getMessage()); } if (connection == null) continue; - log.log(LogLevel.DEBUG, "Got a status page request."); + log.log(Level.FINE, "Got a status page request."); String requestString = ""; OutputStream output = null; try (BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream()))) { @@ -168,7 +168,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { if (s == null || s.equals("")) break; sb.append(s).append("\n"); } - log.log(LogLevel.DEBUG, "Got HTTP request: " + sb.toString()); + log.log(Level.FINE, "Got HTTP request: " + sb.toString()); HttpRequest httpRequest = null; StatusPageResponse response = null; @@ -255,7 +255,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { } } } catch (InterruptedException e) { - log.log(LogLevel.DEBUG, "Status processing thread shut down by interrupt exception: " + e); + log.log(Level.FINE, "Status processing thread shut down by interrupt exception: " + e); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java index 8d8835d5932..d6959ed1952 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java @@ -84,7 +84,7 @@ public class DummyVdsNode { private Thread messageResponder = new Thread() { public void run() { - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": starting message reponder thread"); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": starting message reponder thread"); while (true) { synchronized (timer) { if (isInterrupted()) break; @@ -92,7 +92,7 @@ public class DummyVdsNode { for (Iterator it = waitingRequests.iterator(); it.hasNext(); ) { Req r = it.next(); if (r.timeout <= currentTime) { - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state request at time " + currentTime); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state request at time " + currentTime); r.request.returnValues().add(new StringValue(nodeState.serialize())); if (r.request.methodName().equals("getnodestate3")) { r.request.returnValues().add(new StringValue("No host info in dummy implementation")); @@ -109,7 +109,7 @@ public class DummyVdsNode { } } } - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": shut down message reponder thread"); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": shut down message reponder thread"); } }; @@ -171,7 +171,7 @@ public class DummyVdsNode { void disconnectBreakConnection() { disconnect(true, FleetControllerTest.timeoutMS, false); } void disconnectAsShutdown() { disconnect(true, FleetControllerTest.timeoutMS, true); } private void disconnect(boolean waitForPendingNodeStateRequest, long timeoutms, boolean setStoppingStateFirst) { - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": Breaking connection." + (waitForPendingNodeStateRequest ? " Waiting for pending state first." : "")); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": Breaking connection." + (waitForPendingNodeStateRequest ? " Waiting for pending state first." : "")); if (waitForPendingNodeStateRequest) { this.waitForPendingGetNodeStateRequest(timeoutms); } @@ -190,7 +190,7 @@ public class DummyVdsNode { acceptor.shutdown().join(); supervisor.transport().shutdown().join(); supervisor = null; - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": Done breaking connection."); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": Done breaking connection."); } public String toString() { @@ -244,25 +244,25 @@ public class DummyVdsNode { private void waitForPendingGetNodeStateRequest(long timeout) { long startTime = System.currentTimeMillis(); long endTime = startTime + timeout; - log.log(LogLevel.DEBUG, "Dummy node " + this + " waiting for pending node state request."); + log.log(Level.FINE, "Dummy node " + this + " waiting for pending node state request."); while (true) { synchronized(timer) { if (!waitingRequests.isEmpty()) { - log.log(LogLevel.DEBUG, "Dummy node " + this + " has pending request, returning."); + log.log(Level.FINE, "Dummy node " + this + " has pending request, returning."); return; } try{ - log.log(LogLevel.DEBUG, "Dummy node " + this + " waiting " + (endTime - startTime) + " ms for pending request."); + log.log(Level.FINE, "Dummy node " + this + " waiting " + (endTime - startTime) + " ms for pending request."); timer.wait(endTime - startTime); } catch (InterruptedException e) { } - log.log(LogLevel.DEBUG, "Dummy node " + this + " woke up to recheck."); + log.log(Level.FINE, "Dummy node " + this + " woke up to recheck."); } startTime = System.currentTimeMillis(); if (startTime >= endTime) { - log.log(LogLevel.DEBUG, "Dummy node " + this + " timeout passed. Don't have pending request."); + log.log(Level.FINE, "Dummy node " + this + " timeout passed. Don't have pending request."); if (!waitingRequests.isEmpty()) { - log.log(LogLevel.DEBUG, "Dummy node " + this + ". Non-empty set of waiting requests"); + log.log(Level.FINE, "Dummy node " + this + ". Non-empty set of waiting requests"); } throw new IllegalStateException("Timeout. No pending get node state request pending after waiting " + timeout + " milliseconds."); } @@ -271,7 +271,7 @@ public class DummyVdsNode { void replyToPendingNodeStateRequests() { for(Req req : waitingRequests) { - log.log(LogLevel.DEBUG, "Dummy node " + this + " answering pending node state request."); + log.log(Level.FINE, "Dummy node " + this + " answering pending node state request."); req.request.returnValues().add(new StringValue(nodeState.serialize())); if (req.request.methodName().equals("getnodestate3")) { req.request.returnValues().add(new StringValue(hostInfo)); @@ -283,7 +283,7 @@ public class DummyVdsNode { } public void setNodeState(NodeState state, String hostInfo) { - log.log(LogLevel.DEBUG, "Dummy node " + this + " got new state: " + state); + log.log(Level.FINE, "Dummy node " + this + " got new state: " + state); synchronized(timer) { this.nodeState = state; this.hostInfo = hostInfo; @@ -415,7 +415,7 @@ public class DummyVdsNode { return; } String stateString = nodeState.serialize(-1, true); - log.log(LogLevel.DEBUG, "Dummy node " + this + " got old type get node state request, answering: " + stateString); + log.log(Level.FINE, "Dummy node " + this + " got old type get node state request, answering: " + stateString); req.returnValues().add(new Int32Value(1)); req.returnValues().add(new StringValue("")); req.returnValues().add(new StringValue(stateString)); @@ -427,7 +427,7 @@ public class DummyVdsNode { for (Iterator it = waitingRequests.iterator(); it.hasNext(); ) { Req r = it.next(); if (r.request.parameters().size() > 2 && r.request.parameters().get(2).asInt32() == index) { - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state reply from controller " + index + " as we received new one"); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state reply from controller " + index + " as we received new one"); r.request.returnValues().add(new StringValue(nodeState.serialize())); r.request.returnValues().add(new StringValue("No host info from dummy implementation")); r.request.returnRequest(); @@ -440,7 +440,7 @@ public class DummyVdsNode { } private void rpc_getNodeState2(Request req) { - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Got " + req.methodName() + " request"); + log.log(Level.FINE, "Dummy node " + this + ": Got " + req.methodName() + " request"); try{ String oldState = req.parameters().get(0).asString(); int timeout = req.parameters().get(1).asInt32(); @@ -452,14 +452,14 @@ public class DummyVdsNode { boolean sentReply = sendGetNodeStateReply(index); NodeState givenState = (oldState.equals("unknown") ? null : NodeState.deserialize(type, oldState)); if (givenState != null && (givenState.equals(nodeState) || sentReply)) { - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Has same state as reported " + givenState + ". Queing request. Timeout is " + timeout + " ms. " + log.log(Level.FINE, "Dummy node " + this + ": Has same state as reported " + givenState + ". Queing request. Timeout is " + timeout + " ms. " + "Will be answered at time " + (timer.getCurrentTimeInMillis() + timeout * 800l / 1000)); req.detach(); waitingRequests.add(new Req(req, timer.getCurrentTimeInMillis() + timeout * 800l / 1000)); - log.log(LogLevel.DEBUG, "Dummy node " + this + " has now " + waitingRequests.size() + " entries and is " + (waitingRequests.isEmpty() ? "empty" : "not empty")); + log.log(Level.FINE, "Dummy node " + this + " has now " + waitingRequests.size() + " entries and is " + (waitingRequests.isEmpty() ? "empty" : "not empty")); timer.notifyAll(); } else { - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Request had " + (givenState == null ? "no state" : "different state(" + givenState +")") + ". Answering with " + nodeState); + log.log(Level.FINE, "Dummy node " + this + ": Request had " + (givenState == null ? "no state" : "different state(" + givenState +")") + ". Answering with " + nodeState); req.returnValues().add(new StringValue(nodeState.serialize())); if (req.methodName().equals("getnodestate3")) { req.returnValues().add(new StringValue("Dummy node host info")); @@ -526,7 +526,7 @@ public class DummyVdsNode { } req.returnValues().add(new Int32Value(1)); req.returnValues().add(new StringValue("OK")); - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Got new system state (through old setsystemstate call) " + newState); + log.log(Level.FINE, "Dummy node " + this + ": Got new system state (through old setsystemstate call) " + newState); } catch (Exception e) { log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering setsystemstate request: " + e.getMessage()); e.printStackTrace(System.err); @@ -550,7 +550,7 @@ public class DummyVdsNode { } timer.notifyAll(); } - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Got new system state " + newState); + log.log(Level.FINE, "Dummy node " + this + ": Got new system state " + newState); } catch (Exception e) { log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering setsystemstate request: " + e.getMessage()); e.printStackTrace(System.err); @@ -573,7 +573,7 @@ public class DummyVdsNode { } timer.notifyAll(); } - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Got new cluster state " + stateBundle); + log.log(Level.FINE, "Dummy node " + this + ": Got new cluster state " + stateBundle); } catch (Exception e) { log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering setdistributionstates request: " + e.getMessage()); e.printStackTrace(System.err); @@ -596,11 +596,11 @@ public class DummyVdsNode { activatedClusterStateVersion = activateVersion; timer.notifyAll(); } else { - log.log(LogLevel.DEBUG, () -> String.format("Dummy node %s: got a mismatching activation (request version %d, " + + log.log(Level.FINE, () -> String.format("Dummy node %s: got a mismatching activation (request version %d, " + "actual %d), not marking version as active", this, activateVersion, actualVersion)); } } - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Activating cluster state version " + activateVersion); + log.log(Level.FINE, "Dummy node " + this + ": Activating cluster state version " + activateVersion); } catch (Exception e) { log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering activate_cluster_state_version request: " + e.getMessage()); e.printStackTrace(System.err); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java index bcf98d003a6..bda24b826fd 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java @@ -23,7 +23,7 @@ public class FakeTimer implements Timer { public synchronized void advanceTime(long time) { long currentTime = getCurrentTimeInMillis(); this.currentTime += time; - log.log(LogLevel.DEBUG, "Time advanced by " + time + " ms. Time increased from " + currentTime + " to " + (currentTime + time)); + log.log(Level.FINE, "Time advanced by " + time + " ms. Time increased from " + currentTime + " to " + (currentTime + time)); notifyAll(); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index d69725ebe20..6d8afa14f3e 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -135,13 +135,13 @@ public abstract class FleetControllerTest implements Waiter { } void setUpSystem(boolean useFakeTimer, FleetControllerOptions options) throws Exception { - log.log(LogLevel.DEBUG, "Setting up system"); + log.log(Level.FINE, "Setting up system"); slobrok = new Slobrok(); this.options = options; if (options.zooKeeperServerAddress != null) { zooKeeperServer = new ZooKeeperTestServer(); this.options.zooKeeperServerAddress = zooKeeperServer.getAddress(); - log.log(LogLevel.DEBUG, "Set up new zookeeper server at " + this.options.zooKeeperServerAddress); + log.log(Level.FINE, "Set up new zookeeper server at " + this.options.zooKeeperServerAddress); } this.options.slobrokConnectionSpecs = new String[1]; this.options.slobrokConnectionSpecs[0] = "tcp/localhost:" + slobrok.port(); diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java index 02862207e53..6a946e1ce75 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java @@ -54,7 +54,7 @@ class OverrideProcessor implements PreProcessor { } public Document process(Document input) throws TransformerException { - log.log(LogLevel.DEBUG, "Preprocessing overrides with " + environment + "." + region); + log.log(Level.FINE, "Preprocessing overrides with " + environment + "." + region); Document ret = Xml.copyDocument(input); Element root = ret.getDocumentElement(); applyOverrides(root, Context.empty()); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java index 521bc141b2b..31349967462 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java @@ -41,7 +41,7 @@ public class FilesApplicationFile extends ApplicationFile { @Override public ApplicationFile delete() { - log.log(LogLevel.DEBUG, "Delete " + file); + log.log(Level.FINE, "Delete " + file); if (file.isDirectory() && !listFiles().isEmpty()) { throw new RuntimeException("files. Can't delete, directory not empty: " + this + "(" + listFiles() + ")." + listFiles().size()); } @@ -154,7 +154,7 @@ public class FilesApplicationFile extends ApplicationFile { private void writeMetaFile(String data, String status) throws IOException { File metaDir = createMetaDir(); - log.log(LogLevel.DEBUG, "meta dir=" + metaDir); + log.log(Level.FINE, "meta dir=" + metaDir); File metaFile = new File(metaDir + "/" + getPath().getName()); if (status == null) { status = ApplicationFile.ContentStatusNew; @@ -174,7 +174,7 @@ public class FilesApplicationFile extends ApplicationFile { private File createMetaDir() { File metaDir = getMetaDir(); if (!metaDir.exists()) { - log.log(LogLevel.DEBUG, "Creating meta dir " + metaDir); + log.log(Level.FINE, "Creating meta dir " + metaDir); metaDir.mkdirs(); } return metaDir; @@ -188,7 +188,7 @@ public class FilesApplicationFile extends ApplicationFile { public MetaData getMetaData() { File metaDir = getMetaDir(); File metaFile = new File(metaDir + "/" + getPath().getName()); - log.log(LogLevel.DEBUG, "Getting metadata for " + metaFile); + log.log(Level.FINE, "Getting metadata for " + metaFile); if (metaFile.exists()) { try { return mapper.readValue(metaFile, MetaData.class); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index c2ad0817de5..79bf3f12649 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -282,7 +282,7 @@ public class FilesApplicationPackage implements ApplicationPackage { String dir = include.getAttribute(IncludeDirs.DIR); validateIncludeDir(dir); IncludeDirs.validateFilesInIncludedDir(dir, include.getParentNode(), this); - log.log(LogLevel.DEBUG, "Adding user include dir '" + dir + "'"); + log.log(Level.FINE, "Adding user include dir '" + dir + "'"); userIncludeDirs.add(dir); } @@ -458,7 +458,7 @@ public class FilesApplicationPackage implements ApplicationPackage { private void addAllDefsFromConfigDir(Map defs, File configDefsDir) { if (! configDefsDir.isDirectory()) return; - log.log(LogLevel.DEBUG, "Getting all config definitions from '" + configDefsDir + "'"); + log.log(Level.FINE, "Getting all config definitions from '" + configDefsDir + "'"); for (File def : configDefsDir.listFiles((File dir, String name) -> name.matches(".*\\.def"))) { String[] nv = def.getName().split("\\.def"); ConfigDefinitionKey key; diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java index 23fe0109871..28f74b4adc0 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java @@ -110,10 +110,10 @@ public class SchemaValidators { if (uris == null) throw new IllegalArgumentException("Could not find XML schemas "); File tmpDir = createTempDirectory(tmpBase.toPath(), "vespa").toFile(); - log.log(LogLevel.DEBUG, "Will save all XML schemas for " + vespaVersion + " to " + tmpDir); + log.log(Level.FINE, "Will save all XML schemas for " + vespaVersion + " to " + tmpDir); while (uris.hasMoreElements()) { URL u = uris.nextElement(); - log.log(LogLevel.DEBUG, "uri for resource 'schema'=" + u.toString()); + log.log(Level.FINE, "uri for resource 'schema'=" + u.toString()); // TODO: When is this the case? Remove? if ("jar".equals(u.getProtocol())) { JarURLConnection jarConnection = (JarURLConnection) u.openConnection(); @@ -127,7 +127,7 @@ public class SchemaValidators { jarFile.close(); } else if ("bundle".equals(u.getProtocol())) { Bundle bundle = getBundle(schemaValidatorClass); - log.log(LogLevel.DEBUG, "bundle=" + bundle); + log.log(Level.FINE, "bundle=" + bundle); // TODO: Hack to handle cases where bundle=null (which seems to always be the case with config-model-fat-amended.jar) if (bundle == null) { String pathPrefix = getDefaults().underVespaHome("share/vespa/schema/"); @@ -135,10 +135,10 @@ public class SchemaValidators { // Fallback to path without version if path with version does not exist if (! schemaPath.exists()) schemaPath = new File(pathPrefix); - log.log(LogLevel.DEBUG, "Using schemas found in " + schemaPath); + log.log(Level.FINE, "Using schemas found in " + schemaPath); copySchemas(schemaPath, tmpDir); } else { - log.log(LogLevel.DEBUG, String.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle + log.log(Level.FINE, String.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle .getVersion())); for (Enumeration entries = bundle.findEntries("schema", "*.rnc", true); entries.hasMoreElements(); ) { diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 2925b394c83..454da0d657a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -266,7 +266,7 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter // TODO: Doctoring on the XML is the wrong level for this. We should be able to mark a model as default instead -Jon private static Element getImplicitAdmin(DeployState deployState) throws IOException, SAXException { String defaultAdminElement = deployState.isHosted() ? getImplicitAdminV4() : getImplicitAdminV2(); - log.log(LogLevel.DEBUG, "No defined, using " + defaultAdminElement); + log.log(Level.FINE, "No defined, using " + defaultAdminElement); return XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(defaultAdminElement))).getDocumentElement(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java index 107908c91db..93f308a2c2e 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java @@ -121,7 +121,7 @@ public final class XmlHelper { public static synchronized DocumentBuilder getDocumentBuilder() { try { DocumentBuilder docBuilder = factory.newDocumentBuilder(); - log.log(LogLevel.DEBUG, "XML parser now operational!"); + log.log(Level.FINE, "XML parser now operational!"); return docBuilder; } catch (ParserConfigurationException e) { log.log(LogLevel.WARNING, "No XML parser available - " + e); diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 43eeb613e6b..fc602ca8dcd 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -203,7 +203,7 @@ public abstract class AbstractConfigProducer "Trying to get config for " + builder.getClass().getDeclaringClass().getName() + + log.log(Level.FINE, () -> "Trying to get config for " + builder.getClass().getDeclaringClass().getName() + " for config id " + quote(configProducer.getConfigId()) + ", found=" + found + ", foundOverride=" + foundOverride); @@ -400,7 +400,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri ConfigInstance.Builder builder = resolveToBuilder(configKey); // TODO: remove if-statement, the builder can never be null. if (builder != null) { - log.log(LogLevel.DEBUG, () -> "Found builder for " + configKey); + log.log(Level.FINE, () -> "Found builder for " + configKey); ConfigPayload payload; InnerCNode innerCNode = targetDef != null ? targetDef.getCNode() : null; if (builder instanceof GenericConfig.GenericConfigBuilder) { @@ -429,7 +429,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private ConfigPayload getConfigFromBuilder(ConfigInstance.Builder builder, InnerCNode targetDef) { try { ConfigInstance instance = InstanceResolver.resolveToInstance(builder, targetDef); - log.log(LogLevel.DEBUG, () -> "getConfigFromBuilder for builder " + builder.getClass().getName() + ", instance=" + instance); + log.log(Level.FINE, () -> "getConfigFromBuilder for builder " + builder.getClass().getName() + ", instance=" + instance); return ConfigPayload.fromInstance(instance); } catch (ConfigurationRuntimeException e) { // This can happen in cases where services ask for config that no longer exist before they have been able @@ -465,12 +465,12 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri final String builderName = fullClassName + "$Builder"; if (classLoader == null) { classLoader = getClass().getClassLoader(); - log.log(LogLevel.DEBUG, () -> "No producer found to get classloader from for " + fullClassName + ". Using default"); + log.log(Level.FINE, () -> "No producer found to get classloader from for " + fullClassName + ". Using default"); } try { clazz = classLoader.loadClass(builderName); } catch (ClassNotFoundException e) { - log.log(LogLevel.DEBUG, () -> "Tried to load " + builderName + ", not found, trying with generic builder"); + log.log(Level.FINE, () -> "Tried to load " + builderName + ", not found, trying with generic builder"); // TODO: Enable config compiler when configserver is using new API. // ConfigCompiler compiler = new LazyConfigCompiler(Files.createTempDir()); // return compiler.compile(targetDef.generateClass()).newInstance(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 71e90686e22..8437df43df9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -89,7 +89,7 @@ public class RankSetupValidator extends Validator { // Give up, don't say same error msg repeatedly deleteTempDir(tempDir); } - log.log(LogLevel.DEBUG, String.format("Validating %s for %s, %s took %s ms", + log.log(Level.FINE, String.format("Validating %s for %s, %s took %s ms", sdName, searchCluster, configId, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java index 92785e00436..03b09a2bf2d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java @@ -26,7 +26,7 @@ public class UserConfigBuilder { if (producerSpec == null) { log.log(LogLevel.SPAM, "In getUserConfigs. producerSpec is null"); } - log.log(LogLevel.DEBUG, "getUserConfigs for " + producerSpec); + log.log(Level.FINE, "getUserConfigs for " + producerSpec); for (Element configE : XML.getChildren(producerSpec, "config")) { buildElement(configE, builderMap, configDefinitionStore, deployLogger); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 9067df53a91..9ca5904a7c5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -136,7 +136,7 @@ public class VespaDomBuilder extends VespaModelBuilder { UserConfigRepo userConfigs = UserConfigBuilder.build(producerSpec, deployState, deployState.getDeployLogger()); // TODO: must be made to work: //userConfigs.applyWarnings(child); - log.log(LogLevel.DEBUG, "Adding user configs " + userConfigs + " for " + producerSpec); + log.log(Level.FINE, "Adding user configs " + userConfigs + " for " + producerSpec); child.mergeUserConfigs(userConfigs); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index e27c26bd248..c8b16f5376a 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -50,7 +50,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer public void run() { try { Acceptor acceptor = supervisor.listen(spec); - log.log(LogLevel.DEBUG, "Ready for requests on " + spec); + log.log(Level.FINE, "Ready for requests on " + spec); supervisor.transport().join(); acceptor.shutdown().join(); } catch (ListenFailedException e) { @@ -270,7 +270,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer */ private void getConfigImpl(JRTServerConfigRequest request) { request.getRequestTrace().trace(TRACELEVEL, "Config proxy getConfig()"); - log.log(LogLevel.DEBUG, () ->"getConfig: " + request.getShortDescription() + ",configmd5=" + request.getRequestConfigMd5()); + log.log(Level.FINE, () ->"getConfig: " + request.getShortDescription() + ",configmd5=" + request.getRequestConfigMd5()); if (!request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. log.log(LogLevel.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); @@ -335,12 +335,12 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer */ @Override public void notifyTargetInvalid(Target target) { - log.log(LogLevel.DEBUG, () -> "Target invalid " + target); + log.log(Level.FINE, () -> "Target invalid " + target); for (Iterator it = proxyServer.delayedResponses().responses().iterator(); it.hasNext(); ) { DelayedResponse delayed = it.next(); JRTServerConfigRequest request = delayed.getRequest(); if (request.getRequest().target().equals(target)) { - log.log(LogLevel.DEBUG, () -> "Removing " + request.getShortDescription()); + log.log(Level.FINE, () -> "Removing " + request.getShortDescription()); it.remove(); } } @@ -351,7 +351,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer public void returnOkResponse(JRTServerConfigRequest request, RawConfig config) { request.getRequestTrace().trace(TRACELEVEL, "Config proxy returnOkResponse()"); request.addOkResponse(config.getPayload(), config.getGeneration(), config.isInternalRedeploy(), config.getConfigMd5()); - log.log(LogLevel.DEBUG, () -> "Return response: " + request.getShortDescription() + ",configMd5=" + config.getConfigMd5() + + log.log(Level.FINE, () -> "Return response: " + request.getShortDescription() + ",configMd5=" + config.getConfigMd5() + ",generation=" + config.getGeneration()); log.log(LogLevel.SPAM, () -> "Config payload in response for " + request.getShortDescription() + ":" + config.getPayload()); @@ -361,7 +361,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer try { request.getRequest().returnRequest(); } catch (IllegalStateException e) { - log.log(LogLevel.DEBUG, () -> "Something bad happened when sending response for '" + request.getShortDescription() + "':" + e.getMessage()); + log.log(Level.FINE, () -> "Something bad happened when sending response for '" + request.getShortDescription() + "':" + e.getMessage()); } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java index e10289733df..50f8ad26bf0 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java @@ -50,7 +50,7 @@ public class MemoryCache { return; } - log.log(LogLevel.DEBUG, () -> "Putting '" + config + "' into memory cache"); + log.log(Level.FINE, () -> "Putting '" + config + "' into memory cache"); cache.put(new ConfigCacheKey(config.getKey(), config.getDefMd5()), config); } @@ -114,11 +114,11 @@ public class MemoryCache { try { filename = path + File.separator + createCacheFileName(config); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Writing '" + config.getKey() + "' to '" + filename + "'"); + log.log(Level.FINE, "Writing '" + config.getKey() + "' to '" + filename + "'"); } final Payload payload = config.getPayload(); long protocolVersion = 3; - log.log(LogLevel.DEBUG, "Writing config '" + config + "' to file '" + filename + "' with protocol version " + protocolVersion); + log.log(Level.FINE, "Writing config '" + config + "' to file '" + filename + "' with protocol version " + protocolVersion); writer = IOUtils.createWriter(filename, "UTF-8", false); // First three lines are meta-data about config as comment lines, fourth line is empty diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java index 16b80f8d31e..df0b274f32b 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java @@ -32,11 +32,11 @@ class MemoryCacheConfigClient implements ConfigSourceClient { */ @Override public RawConfig getConfig(RawConfig input, JRTServerConfigRequest request) { - log.log(LogLevel.DEBUG, () -> "Getting config from cache"); + log.log(Level.FINE, () -> "Getting config from cache"); ConfigKey key = input.getKey(); RawConfig cached = cache.get(new ConfigCacheKey(key, input.getDefMd5())); if (cached != null) { - log.log(LogLevel.DEBUG, () -> "Found config " + key + " in cache"); + log.log(Level.FINE, () -> "Found config " + key + " in cache"); return cached; } else { return null; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index ddc0c0d8320..1ebf3106a94 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -48,7 +48,7 @@ public class ProxyServer implements Runnable { ProxyServer(Spec spec, ConfigSourceSet source, MemoryCache memoryCache, ConfigSourceClient configClient) { this.configSource = source; - log.log(LogLevel.DEBUG, "Using config source '" + source); + log.log(Level.FINE, "Using config source '" + source); this.memoryCache = memoryCache; this.rpcServer = createRpcServer(spec); this.configClient = (configClient == null) ? createRpcClient(rpcServer, source, memoryCache) : configClient; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index ddf3bc4d9ce..d37408f97b3 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -92,7 +92,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { Target target = supervisor.connect(spec); target.invokeSync(req, 30.0); if (target.isValid()) { - log.log(LogLevel.DEBUG, () -> "Created connection to config source at " + spec.toString()); + log.log(Level.FINE, () -> "Created connection to config source at " + spec.toString()); return; } else { log.log(LogLevel.INFO, "Could not connect to config source at " + spec.toString()); @@ -132,7 +132,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { RawConfig ret = null; if (cachedConfig != null) { - log.log(LogLevel.DEBUG, () -> "Found config " + configCacheKey + " in cache, generation=" + cachedConfig.getGeneration() + + log.log(Level.FINE, () -> "Found config " + configCacheKey + " in cache, generation=" + cachedConfig.getGeneration() + ",configmd5=" + cachedConfig.getConfigMd5()); log.log(LogLevel.SPAM, () -> "input config=" + input + ",cached config=" + cachedConfig); if (ProxyServer.configOrGenerationHasChanged(cachedConfig, request)) { @@ -155,9 +155,9 @@ class RpcConfigSourceClient implements ConfigSourceClient { private void subscribeToConfig(RawConfig input, ConfigCacheKey configCacheKey) { synchronized (activeSubscribersLock) { if (activeSubscribers.containsKey(configCacheKey)) { - log.log(LogLevel.DEBUG, () -> "Already a subscriber running for: " + configCacheKey); + log.log(Level.FINE, () -> "Already a subscriber running for: " + configCacheKey); } else { - log.log(LogLevel.DEBUG, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey); + log.log(Level.FINE, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey); UpstreamConfigSubscriber subscriber = new UpstreamConfigSubscriber(input, this, configSourceSet, timingValues, requester, memoryCache); try { @@ -215,14 +215,14 @@ class RpcConfigSourceClient implements ConfigSourceClient { * @param config new config */ public void updateSubscribers(RawConfig config) { - log.log(LogLevel.DEBUG, () -> "Config updated for " + config.getKey() + "," + config.getGeneration()); + log.log(Level.FINE, () -> "Config updated for " + config.getKey() + "," + config.getGeneration()); DelayQueue responseDelayQueue = delayedResponses.responses(); log.log(LogLevel.SPAM, () -> "Delayed response queue: " + responseDelayQueue); if (responseDelayQueue.size() == 0) { - log.log(LogLevel.DEBUG, () -> "There exists no matching element on delayed response queue for " + config.getKey()); + log.log(Level.FINE, () -> "There exists no matching element on delayed response queue for " + config.getKey()); return; } else { - log.log(LogLevel.DEBUG, () -> "Delayed response queue has " + responseDelayQueue.size() + " elements"); + log.log(Level.FINE, () -> "Delayed response queue has " + responseDelayQueue.size() + " elements"); } boolean found = false; for (DelayedResponse response : responseDelayQueue.toArray(new DelayedResponse[0])) { @@ -232,7 +232,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { && (config.getGeneration() >= request.getRequestGeneration() || config.getGeneration() == 0)) { if (delayedResponses.remove(response)) { found = true; - log.log(LogLevel.DEBUG, () -> "Call returnOkResponse for " + config.getKey() + "," + config.getGeneration()); + log.log(Level.FINE, () -> "Call returnOkResponse for " + config.getKey() + "," + config.getGeneration()); rpcServer.returnOkResponse(request, config); } else { log.log(LogLevel.INFO, "Could not remove " + config.getKey() + " from delayedResponses queue, already removed"); @@ -240,9 +240,9 @@ class RpcConfigSourceClient implements ConfigSourceClient { } } if (!found) { - log.log(LogLevel.DEBUG, () -> "Found no recipient for " + config.getKey() + " in delayed response queue"); + log.log(Level.FINE, () -> "Found no recipient for " + config.getKey() + " in delayed response queue"); } - log.log(LogLevel.DEBUG, () -> "Finished updating config for " + config.getKey() + "," + config.getGeneration()); + log.log(Level.FINE, () -> "Finished updating config for " + config.getKey() + "," + config.getGeneration()); } @Override diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java index c7567c45c27..14d93ee046e 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java @@ -65,7 +65,7 @@ public class UpstreamConfigSubscriber implements Subscriber { private void updateWithNewConfig(GenericConfigHandle handle) { RawConfig newConfig = handle.getRawConfig(); - log.log(LogLevel.DEBUG, () -> "config to be returned for '" + newConfig.getKey() + + log.log(Level.FINE, () -> "config to be returned for '" + newConfig.getKey() + "', generation=" + newConfig.getGeneration() + ", payload=" + newConfig.getPayload()); memoryCache.update(newConfig); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java index c2daa6e739b..146a5e644ef 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java @@ -63,7 +63,7 @@ class CachedFilesMaintainer implements Runnable { File[] files = directory.listFiles(); if (files != null) filesOnDisk.addAll(Arrays.stream(files).map(File::getName).collect(Collectors.toSet())); - log.log(LogLevel.DEBUG, "Files on disk (in " + directory + "): " + filesOnDisk); + log.log(Level.FINE, "Files on disk (in " + directory + "): " + filesOnDisk); Set filesToDelete = filesOnDisk .stream() diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java index 6f5b125cc1e..3b8a882963c 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java @@ -106,7 +106,7 @@ class FileDistributionRpcServer { } private void setFileReferencesToDownload(Request req) { - log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); Arrays.stream(req.parameters().get(0).asStringArray()) .map(FileReference::new) .forEach(fileReference -> downloader.downloadIfNeeded(new FileReferenceDownload(fileReference))); @@ -115,12 +115,12 @@ class FileDistributionRpcServer { private void downloadFile(Request req) { FileReference fileReference = new FileReference(req.parameters().get(0).asString()); - log.log(LogLevel.DEBUG, () -> "getFile() called for file reference '" + fileReference.value() + "'"); + log.log(Level.FINE, () -> "getFile() called for file reference '" + fileReference.value() + "'"); Optional file = downloader.getFile(fileReference); if (file.isPresent()) { new RequestTracker().trackRequest(file.get().getParentFile()); req.returnValues().add(new StringValue(file.get().getAbsolutePath())); - log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' available at " + file.get()); + log.log(Level.FINE, () -> "File reference '" + fileReference.value() + "' available at " + file.get()); } else { log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found"); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java index 970e4da5511..74393ca07f9 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java @@ -111,7 +111,7 @@ class UrlDownloadRpcServer { writeLastModifiedTimestamp(downloadDir, connection.getLastModified()); new RequestTracker().trackRequest(downloadDir); req.returnValues().add(new StringValue(contentsPath.getAbsolutePath())); - log.log(LogLevel.DEBUG, () -> "URL '" + url + "' available at " + contentsPath); + log.log(Level.FINE, () -> "URL '" + url + "' available at " + contentsPath); log.log(LogLevel.INFO, String.format("Download of URL '%s' done in %.3f seconds", url, (System.currentTimeMillis() -start) / 1000.0)); } else { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 8db1ac5c0a0..be607907854 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -450,14 +450,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye log.log(LogLevel.WARNING, "Getting file references in use for '" + application + "' failed", e); } } - log.log(LogLevel.DEBUG, "File references in use : " + fileReferencesInUse); + log.log(Level.FINE, "File references in use : " + fileReferencesInUse); // Find those on disk that are not in use Set fileReferencesOnDisk = new HashSet<>(); File[] filesOnDisk = fileReferencesPath.listFiles(); if (filesOnDisk != null) fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet())); - log.log(LogLevel.DEBUG, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); + log.log(Level.FINE, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); Instant instant = Instant.now().minus(keepFileReferences); Set fileReferencesToDelete = fileReferencesOnDisk @@ -783,7 +783,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private void cleanupTempDirectory(File tempDir) { - logger.log(LogLevel.DEBUG, "Deleting tmp dir '" + tempDir + "'"); + logger.log(Level.FINE, "Deleting tmp dir '" + tempDir + "'"); if (!IOUtils.recursiveDeleteDir(tempDir)) { logger.log(LogLevel.WARNING, "Not able to delete tmp dir '" + tempDir + "'"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index a3fbe9ec5fb..98c751dcee5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -97,7 +97,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails()); this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server")); - log.log(LogLevel.DEBUG, "Bootstrap mode: " + mode + ", VIP status mode: " + vipStatusMode); + log.log(Level.FINE, "Bootstrap mode: " + mode + ", VIP status mode: " + vipStatusMode); initializing(vipStatusMode); switch (mode) { case BOOTSTRAP_IN_SEPARATE_THREAD: @@ -121,7 +121,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable log.log(LogLevel.INFO, "Stopping config server"); down(); server.stop(); - log.log(LogLevel.DEBUG, "RPC server stopped"); + log.log(Level.FINE, "RPC server stopped"); rpcServerExecutor.shutdown(); serverThread.ifPresent(thread -> { try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java index 72d91e99278..6cf0ad5fab7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java @@ -68,10 +68,10 @@ public class ConfigServerDB { for (String pluginDirectory : pluginDirectories) { bundles.addAll(Bundle.getBundles(new File(pluginDirectory))); } - log.log(LogLevel.DEBUG, "Found " + bundles.size() + " bundles"); + log.log(Level.FINE, "Found " + bundles.size() + " bundles"); List addedBundles = new ArrayList<>(); for (Bundle bundle : bundles) { - log.log(LogLevel.DEBUG, "Bundle in " + bundle.getFile().getAbsolutePath() + " appears to contain " + bundle.getDefEntries().size() + " entries"); + log.log(Level.FINE, "Bundle in " + bundle.getFile().getAbsolutePath() + " appears to contain " + bundle.getDefEntries().size() + " entries"); configDefinitionDir.addConfigDefinitionsFromBundle(bundle, addedBundles); addedBundles.add(bundle); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index aa6c2c1a2b2..6fcfde80510 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -74,10 +74,10 @@ public class SuperModelRequestHandler implements RequestHandler { @Override public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional vespaVersion) { - log.log(LogLevel.DEBUG, () -> "SuperModelRequestHandler resolving " + req + " for app id '" + appId + "'"); + log.log(Level.FINE, () -> "SuperModelRequestHandler resolving " + req + " for app id '" + appId + "'"); if (handler != null) { ConfigResponse configResponse = handler.resolveConfig(req); - log.log(LogLevel.DEBUG, () -> "SuperModelRequestHandler returning response for config " + req + + log.log(Level.FINE, () -> "SuperModelRequestHandler returning response for config " + req + " with generation " + configResponse.getGeneration()); return configResponse; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index ef9c5161e98..10cfcc96a09 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -153,7 +153,7 @@ public class Application implements ModelResult { } private void debug(String message) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(getId())+message); + log.log(Level.FINE, TenantRepository.logPre(getId())+message); } private ConfigDefinition getTargetDef(GetConfigRequest req) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index 931d85dbe45..2cab0ea5e20 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -84,23 +84,23 @@ public class CompressedApplicationInputStream implements AutoCloseable { } private void decompressInto(File application) throws IOException { - log.log(LogLevel.DEBUG, "Application is in " + application.getAbsolutePath()); + log.log(Level.FINE, "Application is in " + application.getAbsolutePath()); int entries = 0; ArchiveEntry entry; while ((entry = ais.getNextEntry()) != null) { - log.log(LogLevel.DEBUG, "Unpacking " + entry.getName()); + log.log(Level.FINE, "Unpacking " + entry.getName()); File outFile = new File(application, entry.getName()); // FIXME/TODO: write more tests that break this logic. I have a feeling it is not very robust. if (entry.isDirectory()) { if (!(outFile.exists() && outFile.isDirectory())) { - log.log(LogLevel.DEBUG, "Creating dir: " + outFile.getAbsolutePath()); + log.log(Level.FINE, "Creating dir: " + outFile.getAbsolutePath()); boolean res = outFile.mkdirs(); if (!res) { log.log(LogLevel.WARNING, "Could not create dir " + entry.getName()); } } } else { - log.log(LogLevel.DEBUG, "Creating output file: " + outFile.getAbsolutePath()); + log.log(Level.FINE, "Creating output file: " + outFile.getAbsolutePath()); // Create parent dir if necessary String parent = outFile.getParent(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index 9ffca3d5ea5..824cc932ecc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -72,7 +72,7 @@ public class ConfigConvergenceChecker extends AbstractComponent { /** Check all services in given application. Returns the minimum current generation of all services */ public ServiceListResponse servicesToCheck(Application application, URI requestUrl, Duration timeoutPerService) { - log.log(LogLevel.DEBUG, () -> "Finding services to check config convergence for in '" + application); + log.log(Level.FINE, () -> "Finding services to check config convergence for in '" + application); List servicesToCheck = new ArrayList<>(); application.getModel().getHosts() .forEach(host -> host.getServices().stream() diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java index 52aae860480..ac994e8405d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java @@ -25,7 +25,7 @@ public class PermanentApplicationPackage { File app = new File(getDefaults().underVespaHome(config.applicationDirectory())); applicationPackage = Optional.ofNullable(app.exists() ? FilesApplicationPackage.fromFile(app) : null); if (applicationPackage.isPresent()) { - log.log(LogLevel.DEBUG, "Detected permanent application package in '" + + log.log(Level.FINE, "Detected permanent application package in '" + getDefaults().underVespaHome(config.applicationDirectory()) + "'. This might add extra services to config models"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index a1b7df5f19d..318cd367e30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -175,7 +175,7 @@ public class TenantApplications { } private void applicationAdded(ApplicationId applicationId) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(applicationId) + "Application added: " + applicationId); + log.log(Level.FINE, TenantRepository.logPre(applicationId) + "Application added: " + applicationId); } private Path applicationPath(ApplicationId id) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 5f2bece849b..37dd0502ee8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -196,12 +196,12 @@ public class Deployment implements com.yahoo.config.provision.Deployment { private void checkIfActiveHasChanged(LocalSession session, LocalSession currentActiveSession, boolean ignoreStaleSessionFailure) { long activeSessionAtCreate = session.getActiveSessionAtCreate(); - log.log(LogLevel.DEBUG, currentActiveSession.logPre() + "active session id at create time=" + activeSessionAtCreate); + log.log(Level.FINE, currentActiveSession.logPre() + "active session id at create time=" + activeSessionAtCreate); if (activeSessionAtCreate == 0) return; // No active session at create long sessionId = session.getSessionId(); long currentActiveSessionSessionId = currentActiveSession.getSessionId(); - log.log(LogLevel.DEBUG, currentActiveSession.logPre() + "sessionId=" + sessionId + + log.log(Level.FINE, currentActiveSession.logPre() + "sessionId=" + sessionId + ", current active session=" + currentActiveSessionSessionId); if (currentActiveSession.isNewerThan(activeSessionAtCreate) && currentActiveSessionSessionId != sessionId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 25747113d9e..c4279414a9b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -82,7 +82,7 @@ public class FileDirectory { if (file.isDirectory()) { return Files.walk(file.toPath(), 100).map(path -> { try { - log.log(LogLevel.DEBUG, "Calculating hash for '" + path + "'"); + log.log(Level.FINE, "Calculating hash for '" + path + "'"); return hash(path.toFile(), hasher); } catch (IOException e) { log.log(LogLevel.WARNING, "Failed getting hash from '" + path + "'"); @@ -143,15 +143,15 @@ public class FileDirectory { File destination = new File(tempDestinationDir.toFile(), source.getName()); if (!destinationDir.exists()) { destinationDir.mkdir(); - log.log(LogLevel.DEBUG, "file reference ' " + reference.value() + "', source: " + source.getAbsolutePath() ); + log.log(Level.FINE, "file reference ' " + reference.value() + "', source: " + source.getAbsolutePath() ); if (source.isDirectory()) { - log.log(LogLevel.DEBUG, "Copying source " + source.getAbsolutePath() + " to " + destination.getAbsolutePath()); + log.log(Level.FINE, "Copying source " + source.getAbsolutePath() + " to " + destination.getAbsolutePath()); IOUtils.copyDirectory(source, destination, -1); } else { copyFile(source, destination); } if (!destinationDir.exists()) { - log.log(LogLevel.DEBUG, "Moving from " + tempDestinationDir + " to " + destinationDir.getAbsolutePath()); + log.log(Level.FINE, "Moving from " + tempDestinationDir + " to " + destinationDir.getAbsolutePath()); if ( ! tempDestinationDir.toFile().renameTo(destinationDir)) { log.log(LogLevel.WARNING, "Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'."); } @@ -168,7 +168,7 @@ public class FileDirectory { private void logfileInfo(File file ) throws IOException { BasicFileAttributes basicFileAttributes = Files.readAttributes(file.toPath(), BasicFileAttributes.class); - log.log(LogLevel.DEBUG, "Adding file " + file.getAbsolutePath() + " (created " + basicFileAttributes.creationTime() + + log.log(Level.FINE, "Adding file " + file.getAbsolutePath() + " (created " + basicFileAttributes.creationTime() + ", modified " + basicFileAttributes.lastModifiedTime() + ", size " + basicFileAttributes.size() + ")"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java index 3040a43bde4..56bdeab76b2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java @@ -49,7 +49,7 @@ public class FileDistributionImpl implements FileDistribution, RequestWaiter { Request request = new Request("filedistribution.setFileReferencesToDownload"); request.setContext(target); request.parameters().add(new StringArray(fileReferences.stream().map(FileReference::value).toArray(String[]::new))); - log.log(LogLevel.DEBUG, "Executing " + request.methodName() + " against " + target); + log.log(Level.FINE, "Executing " + request.methodName() + " against " + target); target.invokeAsync(request, rpcTimeout, this); } @@ -58,7 +58,7 @@ public class FileDistributionImpl implements FileDistribution, RequestWaiter { public void handleRequestDone(Request req) { Target target = (Target) req.getContext(); if (req.isError()) { - log.log(LogLevel.DEBUG, req.methodName() + " failed for " + target + ": " + req.errorCode() + " (" + req.errorMessage() + ")"); + log.log(Level.FINE, req.methodName() + " failed for " + target + ": " + req.errorCode() + " (" + req.errorMessage() + ")"); } if (target != null) target.close(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 465c8a02200..6cde8cca18f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -101,7 +101,7 @@ public class FileServer { try { return root.getFile(reference).exists(); } catch (IllegalArgumentException e) { - log.log(LogLevel.DEBUG, "Failed locating file reference '" + reference + "' with error " + e.toString()); + log.log(Level.FINE, "Failed locating file reference '" + reference + "' with error " + e.toString()); } return false; } @@ -117,7 +117,7 @@ public class FileServer { private void serveFile(FileReference reference, Receiver target) { File file = root.getFile(reference); - log.log(LogLevel.DEBUG, () -> "Start serving reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); + log.log(Level.FINE, () -> "Start serving reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); boolean success = false; String errorDescription = "OK"; FileReferenceData fileData = FileReferenceDataBlob.empty(reference, file.getName()); @@ -131,7 +131,7 @@ public class FileServer { try { target.receive(fileData, new ReplayStatus(success ? 0 : 1, success ? "OK" : errorDescription)); - log.log(LogLevel.DEBUG, "Done serving file reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); + log.log(Level.FINE, "Done serving file reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); } catch (Exception e) { log.log(LogLevel.WARNING, "Failed serving file reference '" + reference.value() + "': " + Exceptions.toMessageString(e)); } finally { @@ -156,7 +156,7 @@ public class FileServer { } private void serveFileInternal(String fileReference, boolean downloadFromOtherSourceIfNotFound, Request request, Receiver receiver) { - log.log(LogLevel.DEBUG, () -> "Received request for reference '" + fileReference + "' from " + request.target()); + log.log(Level.FINE, () -> "Received request for reference '" + fileReference + "' from " + request.target()); boolean fileExists; try { @@ -178,10 +178,10 @@ public class FileServer { // This is to avoid config servers asking each other for a file that does not exist private boolean download(String fileReference, boolean downloadFromOtherSourceIfNotFound) { if (downloadFromOtherSourceIfNotFound) { - log.log(LogLevel.DEBUG, "File not found, downloading from another source"); + log.log(Level.FINE, "File not found, downloading from another source"); return download(fileReference).isPresent(); } else { - log.log(LogLevel.DEBUG, "File not found, will not download from another source since request came from another config server"); + log.log(Level.FINE, "File not found, will not download from another source since request came from another config server"); return false; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java index cb33371aee9..ec37f2598e0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java @@ -28,7 +28,7 @@ public class HostRegistry implements HostValidator { public synchronized void update(T key, Collection newHosts) { verifyHosts(key, newHosts); Collection currentHosts = getHostsForKey(key); - log.log(LogLevel.DEBUG, () -> "Setting hosts for key '" + key + "', " + + log.log(Level.FINE, () -> "Setting hosts for key '" + key + "', " + "newHosts: " + newHosts + ", " + "currentHosts: " + currentHosts); Collection removedHosts = getRemovedHosts(newHosts, currentHosts); @@ -71,14 +71,14 @@ public class HostRegistry implements HostValidator { private void removeHosts(Collection removedHosts) { for (String host : removedHosts) { - log.log(LogLevel.DEBUG, () -> "Removing " + host); + log.log(Level.FINE, () -> "Removing " + host); host2KeyMap.remove(host); } } private void addHosts(T key, Collection newHosts) { for (String host : newHosts) { - log.log(LogLevel.DEBUG, () -> "Adding " + host); + log.log(Level.FINE, () -> "Adding " + host); host2KeyMap.put(host, key); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java index 592aed83024..7458c82942f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java @@ -41,7 +41,7 @@ public class HttpGetConfigHandler extends HttpHandler { } private ConfigResponse resolveConfig(HttpConfigRequest request) { - log.log(LogLevel.DEBUG, "nocache=" + request.noCache()); + log.log(Level.FINE, "nocache=" + request.noCache()); ConfigResponse config = requestHandler.resolveConfig(ApplicationId.defaultId(), request, Optional.empty()); if (config == null) HttpConfigRequest.throwModelNotReady(); return config; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index 78bc0f48232..e1a5845c4a1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -32,7 +32,7 @@ public class HttpHandler extends LoggingRequestHandler { @Override public HttpResponse handle(HttpRequest request) { - log.log(LogLevel.DEBUG, request.getMethod() + " " + request.getUri().toString()); + log.log(Level.FINE, request.getMethod() + " " + request.getUri().toString()); try { switch (request.getMethod()) { case POST: diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java index 7fc9c4ead47..075b4bc329b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java @@ -30,7 +30,7 @@ class SessionContentStatusListResponse extends SessionResponse { element.setString("status", f.getMetaData().getStatus()); element.setString("md5", f.getMetaData().getMd5()); element.setString("name", urlBase + f.getPath()); - log.log(LogLevel.DEBUG, "Adding file " + urlBase + f.getPath()); + log.log(Level.FINE, "Adding file " + urlBase + f.getPath()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java index 577a0fdd17a..514e0c1c8ae 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -59,7 +59,7 @@ public class TesterClient { } private HttpResponse execute(HttpUriRequest request, String messageIfRequestFails) { - logger.log(LogLevel.DEBUG, "Sending request to tester container " + request.getURI().toString()); + logger.log(Level.FINE, "Sending request to tester container " + request.getURI().toString()); try { return new ProxyResponse(httpClient.execute(request)); } catch (IOException e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java index ddeb49fda6f..ffe3d39b524 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -39,22 +39,22 @@ public class HostHandler extends HttpHandler { @Override public HttpResponse handleGET(HttpRequest request) { String hostname = getBindingMatch(request).group(2); - log.log(LogLevel.DEBUG, "hostname=" + hostname); + log.log(Level.FINE, "hostname=" + hostname); HostRegistry tenantHostRegistry = hostRegistries.getTenantHostRegistry(); - log.log(LogLevel.DEBUG, "hosts in tenant host registry '" + tenantHostRegistry + "' " + tenantHostRegistry.getAllHosts()); + log.log(Level.FINE, "hosts in tenant host registry '" + tenantHostRegistry + "' " + tenantHostRegistry.getAllHosts()); TenantName tenant = tenantHostRegistry.getKeyForHost(hostname); if (tenant == null) return createError(hostname); - log.log(LogLevel.DEBUG, "tenant=" + tenant); + log.log(Level.FINE, "tenant=" + tenant); HostRegistry applicationIdHostRegistry = hostRegistries.getApplicationHostRegistry(tenant); ApplicationId applicationId; if (applicationIdHostRegistry == null) return createError(hostname); applicationId = applicationIdHostRegistry.getKeyForHost(hostname); - log.log(LogLevel.DEBUG, "applicationId=" + applicationId); + log.log(Level.FINE, "applicationId=" + applicationId); if (applicationId == null) { return createError(hostname); } else { - log.log(LogLevel.DEBUG, "hosts in application host registry '" + applicationIdHostRegistry + "' " + applicationIdHostRegistry.getAllHosts()); + log.log(Level.FINE, "hosts in application host registry '" + applicationIdHostRegistry + "' " + applicationIdHostRegistry.getAllHosts()); return new HostResponse(Response.Status.OK, applicationId, zone); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java index 5b4c32da7e1..b4ce833cb75 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java @@ -41,7 +41,7 @@ public class HttpGetConfigHandler extends HttpHandler { } private ConfigResponse resolveConfig(HttpConfigRequest request, RequestHandler requestHandler) { - log.log(LogLevel.DEBUG, "nocache=" + request.noCache()); + log.log(Level.FINE, "nocache=" + request.noCache()); ConfigResponse config = requestHandler.resolveConfig(request.getApplicationId(), request, Optional.empty()); if (config == null) HttpConfigRequest.throwModelNotReady(); return config; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java index 15ed5fd2408..42d3ace6bd9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java @@ -76,7 +76,7 @@ public class ClusterMetricsRetriever { throw new RuntimeException(e); } - log.log(LogLevel.DEBUG, () -> + log.log(Level.FINE, () -> String.format("Metric retrieval for %d nodes took %d milliseconds", hosts.size(), System.currentTimeMillis() - startTime) ); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index e4276687c38..77d84206fec 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -88,7 +88,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder { Version wantedNodeVespaVersion, Optional ignored, // Ignored since we have this in the app package for activated models Instant now) { - log.log(LogLevel.DEBUG, String.format("Loading model version %s for session %s application %s", + log.log(Level.FINE, String.format("Loading model version %s for session %s application %s", modelFactory.version(), appGeneration, applicationId)); ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId); Provisioned provisioned = new Provisioned(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 2662e14e8e9..361153d2817 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -78,7 +78,7 @@ public abstract class ModelsBuilder { ApplicationPackage applicationPackage, SettableOptional allocatedHosts, Instant now) { - log.log(LogLevel.DEBUG, "Will build models for " + applicationId); + log.log(Level.FINE, "Will build models for " + applicationId); Set versions = modelFactoryRegistry.allVersions(); // If the application specifies a major, skip models on a newer major @@ -130,7 +130,7 @@ public abstract class ModelsBuilder { } } } - log.log(LogLevel.DEBUG, "Done building models for " + applicationId + ". Built models for versions " + + log.log(Level.FINE, "Done building models for " + applicationId + ". Built models for versions " + allApplicationModels.stream() .map(result -> result.getModel().version()) .map(Version::toFullString) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index 032e7232744..34d7d8641ea 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -89,7 +89,7 @@ public class PreparedModelsBuilder extends ModelsBuilder allocatedHosts, Instant now) { Version modelVersion = modelFactory.version(); - log.log(LogLevel.DEBUG, "Building model " + modelVersion + " for " + applicationId); + log.log(Level.FINE, "Building model " + modelVersion + " for " + applicationId); FileDistributionProvider fileDistributionProvider = fileDistributionFactory.createProvider(context.getServerDBSessionDir()); // Use empty on non-hosted systems, use already allocated hosts if available, create connection to a host provisioner otherwise @@ -109,12 +109,12 @@ public class PreparedModelsBuilder extends ModelsBuilder fixedMetric : metricUpdater.getStaticMetrics().entrySet()) { - log.log(LogLevel.DEBUG, "Setting " + fixedMetric.getKey()); + log.log(Level.FINE, "Setting " + fixedMetric.getKey()); metric.set(fixedMetric.getKey(), fixedMetric.getValue(), metricUpdater.getMetricContext()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java index 03b11a89d08..3c18d5f11d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java @@ -89,7 +89,7 @@ public class DelayedConfigResponses { removeWatcher(); rpcServer.addToRequestQueue(request, true, null); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, logPre()+"DelayedConfigResponse. putting on queue: " + request.getShortDescription()); + log.log(Level.FINE, logPre()+"DelayedConfigResponse. putting on queue: " + request.getShortDescription()); } } @@ -169,7 +169,7 @@ public class DelayedConfigResponses { */ final void delayResponse(JRTServerConfigRequest request, GetConfigContext context) { if (request.isDelayedResponse()) { - log.log(LogLevel.DEBUG, context.logPre()+"Request already delayed"); + log.log(Level.FINE, context.logPre()+"Request already delayed"); } else { createQueueIfNotExists(context); BlockingQueue delayedResponsesQueue = delayedResponses.get(context.applicationId()); @@ -177,7 +177,7 @@ public class DelayedConfigResponses { request.setDelayedResponse(true); try { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, context.logPre()+"Putting on delayedRequests queue (" + delayedResponsesQueue.size() + " elements): " + + log.log(Level.FINE, context.logPre()+"Putting on delayedRequests queue (" + delayedResponsesQueue.size() + " elements): " + response.getRequest().getShortDescription()); } // Config will be resolved in the run() method of DelayedConfigResponse, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index ff39faa9d29..f2099388c7c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -164,7 +164,7 @@ class GetConfigProcessor implements Runnable { } private void returnEmpty(JRTServerConfigRequest request) { - log.log(LogLevel.DEBUG, () -> "Returning empty sentinel config for request from " + request.getClientHostName()); + log.log(Level.FINE, () -> "Returning empty sentinel config for request from " + request.getClientHostName()); ConfigPayload emptyPayload = ConfigPayload.empty(); String configMd5 = ConfigUtils.getMd5(emptyPayload); ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, 0, false, configMd5); @@ -178,7 +178,7 @@ class GetConfigProcessor implements Runnable { private void debugLog(Trace trace, String message) { if (logDebug(trace)) { - log.log(LogLevel.DEBUG, logPre + message); + log.log(Level.FINE, logPre + message); trace.trace(RpcServer.TRACELEVEL_DEBUG, logPre + message); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index f2cabe77f36..6082e4d34e5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -263,7 +263,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { List responses = delayedConfigResponses.drainQueue(applicationId); String logPre = TenantRepository.logPre(applicationId); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, logPre + "Start of configReload: " + responses.size() + " requests on delayed requests queue"); + log.log(Level.FINE, logPre + "Start of configReload: " + responses.size() + " requests on delayed requests queue"); } int responsesSent = 0; CompletionService completionService = new ExecutorCompletionService<>(executorService); @@ -281,7 +281,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { responsesSent++; } } else { - log.log(LogLevel.DEBUG, logPre + "Timer already cancelled or finished or never scheduled"); + log.log(Level.FINE, logPre + "Timer already cancelled or finished or never scheduled"); } } @@ -293,7 +293,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } } - log.log(LogLevel.DEBUG, logPre + "Finished reloading " + responsesSent + " requests"); + log.log(Level.FINE, logPre + "Finished reloading " + responsesSent + " requests"); } private void logRequestDebug(LogLevel level, String message, JRTServerConfigRequest request) { @@ -304,7 +304,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Override public void hostsUpdated(TenantName tenant, Collection newHosts) { - log.log(LogLevel.DEBUG, "Updating hosts in tenant host registry '" + hostRegistry + "' with " + newHosts); + log.log(Level.FINE, "Updating hosts in tenant host registry '" + hostRegistry + "' with " + newHosts); hostRegistry.update(tenant, newHosts); } @@ -322,7 +322,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { public void respond(JRTServerConfigRequest request) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Trace at request return:\n" + request.getRequestTrace().toString()); + log.log(Level.FINE, "Trace at request return:\n" + request.getRequestTrace().toString()); } request.getRequest().returnRequest(); } @@ -338,8 +338,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { if (tenant == null) { if (GetConfigProcessor.logDebug(trace)) { String message = "Did not find tenant for host '" + hostname + "', using " + TenantName.defaultName(); - log.log(LogLevel.DEBUG, message); - log.log(LogLevel.DEBUG, "hosts in host registry: " + hostRegistry.getAllHosts()); + log.log(Level.FINE, message); + log.log(Level.FINE, "hosts in host registry: " + hostRegistry.getAllHosts()); trace.trace(6, message); } return Optional.empty(); @@ -420,7 +420,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Override public void onTenantDelete(TenantName tenant) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant)+"Tenant deleted, removing request handler and cleaning host registry"); + log.log(Level.FINE, TenantRepository.logPre(tenant)+"Tenant deleted, removing request handler and cleaning host registry"); tenantProviders.remove(tenant); hostRegistry.removeHostsForKey(tenant); } @@ -433,7 +433,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Override public void onTenantCreate(TenantName tenant, TenantHandlerProvider tenantHandlerProvider) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant)+"Tenant created, adding request handler"); + log.log(Level.FINE, TenantRepository.logPre(tenant)+"Tenant created, adding request handler"); tenantProviders.put(tenant, tenantHandlerProvider); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java index 4a030de8c26..4484fa55057 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -84,7 +84,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { try { getPeerIdentity(request) .ifPresent(peerIdentity -> authorizer.accept(request, peerIdentity)); - log.log(LogLevel.DEBUG, () -> String.format("Authorization succeeded for request '%s' from '%s'", + log.log(Level.FINE, () -> String.format("Authorization succeeded for request '%s' from '%s'", request.methodName(), request.target().toString())); } catch (Throwable t) { handleAuthorizationFailure(request, t); @@ -158,7 +158,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { if (!isAuthorizationException || ((AuthorizationException) throwable).type() != Type.SILENT) { log.log(LogLevel.INFO, errorMessage); } - log.log(LogLevel.DEBUG, throwable, throwable::getMessage); + log.log(Level.FINE, throwable, throwable::getMessage); JrtErrorCode error = isAuthorizationException ? JrtErrorCode.UNAUTHORIZED : JrtErrorCode.AUTHORIZATION_FAILED; request.setError(error.code, errorMessage); request.returnRequest(); @@ -180,7 +180,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { } try { NodeIdentity identity = nodeIdentifier.identifyNode(certChain); - log.log(LogLevel.DEBUG, () -> String.format("Client '%s' identified as %s", request.target().toString(), identity.toString())); + log.log(Level.FINE, () -> String.format("Client '%s' identified as %s", request.target().toString(), identity.toString())); return Optional.of(identity); } catch (NodeIdentifierException e) { throw new AuthorizationException("Failed to identity peer: " + e.getMessage(), e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java index e697e7193ca..bf76cf67cf1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -78,7 +78,7 @@ public class LocalSessionRepo extends SessionRepo { } public void purgeOldSessions() { - log.log(LogLevel.DEBUG, "Purging old sessions"); + log.log(Level.FINE, "Purging old sessions"); try { List sessions = new ArrayList<>(listSessions()); for (LocalSession candidate : sessions) { @@ -90,7 +90,7 @@ public class LocalSessionRepo extends SessionRepo { } catch (Throwable e) { log.log(LogLevel.WARNING, "Error when purging old sessions ", e); } - log.log(LogLevel.DEBUG, "Done purging old sessions"); + log.log(Level.FINE, "Done purging old sessions"); } private boolean hasExpired(LocalSession candidate) { @@ -103,7 +103,7 @@ public class LocalSessionRepo extends SessionRepo { void deleteSession(LocalSession session) { long sessionId = session.getSessionId(); - log.log(LogLevel.DEBUG, "Deleting local session " + sessionId); + log.log(Level.FINE, "Deleting local session " + sessionId); LocalSessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); if (watcher != null) watcher.close(); removeSession(sessionId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java index 6a27a300a15..5a972086d82 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java @@ -41,7 +41,7 @@ public class LocalSessionStateWatcher { session.logPre() + "Session change: Local session " + sessionId + " changed status to " + status); if (status.equals(Session.Status.DELETE) && localSessionRepo.getSession(sessionId) != null) { - log.log(LogLevel.DEBUG, session.logPre() + "Deleting session " + sessionId); + log.log(Level.FINE, session.logPre() + "Deleting session " + sessionId); localSessionRepo.deleteSession(session); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index de084e598c9..0f96d70a3b1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -88,11 +88,11 @@ public class RemoteSession extends Session { void makeActive(ReloadHandler reloadHandler) { Curator.CompletionWaiter waiter = zooKeeperClient.getActiveWaiter(); - log.log(LogLevel.DEBUG, () -> logPre() + "Getting session from repo: " + getSessionId()); + log.log(Level.FINE, () -> logPre() + "Getting session from repo: " + getSessionId()); ApplicationSet app = ensureApplicationLoaded(); - log.log(LogLevel.DEBUG, () -> logPre() + "Reloading config for " + getSessionId()); + log.log(Level.FINE, () -> logPre() + "Reloading config for " + getSessionId()); reloadHandler.reloadConfig(app); - log.log(LogLevel.DEBUG, () -> logPre() + "Notifying " + waiter); + log.log(Level.FINE, () -> logPre() + "Notifying " + waiter); notifyCompletion(waiter); log.log(LogLevel.INFO, logPre() + "Session activated: " + getSessionId()); } @@ -108,9 +108,9 @@ public class RemoteSession extends Session { void confirmUpload() { Curator.CompletionWaiter waiter = zooKeeperClient.getUploadWaiter(); - log.log(LogLevel.DEBUG, "Notifying upload waiter for session " + getSessionId()); + log.log(Level.FINE, "Notifying upload waiter for session " + getSessionId()); notifyCompletion(waiter); - log.log(LogLevel.DEBUG, "Done notifying upload for session " + getSessionId()); + log.log(Level.FINE, "Done notifying upload for session " + getSessionId()); } private void notifyCompletion(Curator.CompletionWaiter completionWaiter) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index 6b5ad0c10a8..e4e097cf448 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -138,7 +138,7 @@ public class RemoteSessionRepo extends SessionRepo { * @param sessionId session id for the new session */ private void sessionAdded(long sessionId) { - log.log(LogLevel.DEBUG, () -> "Adding session to RemoteSessionRepo: " + sessionId); + log.log(Level.FINE, () -> "Adding session to RemoteSessionRepo: " + sessionId); try { RemoteSession session = remoteSessionFactory.createSession(sessionId); Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); @@ -164,7 +164,7 @@ public class RemoteSessionRepo extends SessionRepo { private void loadSessionIfActive(RemoteSession session) { for (ApplicationId applicationId : applicationRepo.activeApplications()) { if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { - log.log(LogLevel.DEBUG, () -> "Found active application for session " + session.getSessionId() + " , loading it"); + log.log(Level.FINE, () -> "Found active application for session " + session.getSessionId() + " , loading it"); reloadHandler.reloadConfig(session.ensureApplicationLoaded()); log.log(LogLevel.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); return; @@ -200,7 +200,7 @@ public class RemoteSessionRepo extends SessionRepo { @SuppressWarnings("unused") private void childEvent(CuratorFramework ignored, PathChildrenCacheEvent event) { zkWatcherExecutor.execute(() -> { - log.log(LogLevel.DEBUG, () -> "Got child event: " + event); + log.log(Level.FINE, () -> "Got child event: " + event); switch (event.getType()) { case CHILD_ADDED: sessionsChanged(); @@ -220,7 +220,7 @@ public class RemoteSessionRepo extends SessionRepo { for (long sessionId : sessionList) { RemoteSession session = getSession(sessionId); if (session == null) continue; // session might have been deleted after getting session list - log.log(LogLevel.DEBUG, () -> session.logPre() + "Confirming upload for session " + sessionId); + log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); session.confirmUpload(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java index 1f11c0f9737..549ad2a2d24 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java @@ -43,11 +43,11 @@ public class RemoteSessionStateWatcher { } private void sessionChanged(Session.Status status) { - log.log(LogLevel.DEBUG, session.logPre() + "Session change: Remote session " + session.getSessionId() + " changed status to " + status); + log.log(Level.FINE, session.logPre() + "Session change: Remote session " + session.getSessionId() + " changed status to " + status); // valid for NEW -> PREPARE transitions, not ACTIVATE -> PREPARE. if (status.equals(Session.Status.PREPARE)) { - log.log(LogLevel.DEBUG, session.logPre() + "Loading prepared session: " + session.getSessionId()); + log.log(Level.FINE, session.logPre() + "Loading prepared session: " + session.getSessionId()); session.loadPrepared(); } else if (status.equals(Session.Status.ACTIVATE)) { session.makeActive(reloadHandler); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index 9473855e2aa..58b643ec82c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -105,16 +105,16 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { SessionZooKeeperClient sessionZKClient, TimeoutBudget timeoutBudget, Clock clock) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Creating session " + sessionId + " in ZooKeeper"); + log.log(Level.FINE, TenantRepository.logPre(tenant) + "Creating session " + sessionId + " in ZooKeeper"); sessionZKClient.createNewSession(clock.instant().toEpochMilli(), TimeUnit.MILLISECONDS); - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Creating upload waiter for session " + sessionId); + log.log(Level.FINE, TenantRepository.logPre(tenant) + "Creating upload waiter for session " + sessionId); Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Done creating upload waiter for session " + sessionId); + log.log(Level.FINE, TenantRepository.logPre(tenant) + "Done creating upload waiter for session " + sessionId); SessionContext context = new SessionContext(applicationPackage, sessionZKClient, getSessionAppDir(sessionId), applicationRepo, hostRegistry, flagSource); LocalSession session = new LocalSession(tenant, sessionId, sessionPreparer, context); - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Waiting on upload waiter for session " + sessionId); + log.log(Level.FINE, TenantRepository.logPre(tenant) + "Waiting on upload waiter for session " + sessionId); waiter.awaitCompletion(timeoutBudget.timeLeft()); - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Done waiting on upload waiter for session " + sessionId); + log.log(Level.FINE, TenantRepository.logPre(tenant) + "Done waiting on upload waiter for session " + sessionId); return session; } @@ -127,7 +127,7 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { ApplicationId existingApplicationId = existingSession.getApplicationId(); long activeSessionId = getActiveSessionId(existingApplicationId); - logger.log(LogLevel.DEBUG, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); + logger.log(Level.FINE, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() session.setApplicationId(existingApplicationId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index d39ddd68f25..c28d432ea08 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -121,7 +121,7 @@ public class SessionPreparer { preparation.writeContainerEndpointsZK(); preparation.distribute(); } - log.log(LogLevel.DEBUG, () -> "time used " + params.getTimeoutBudget().timesUsed() + + log.log(Level.FINE, () -> "time used " + params.getTimeoutBudget().timesUsed() + " : " + params.getApplicationId()); return preparation.result(); } @@ -243,7 +243,7 @@ public class SessionPreparer { } void writeStateZK() { - log.log(LogLevel.DEBUG, "Writing application package state to zookeeper"); + log.log(Level.FINE, "Writing application package state to zookeeper"); writeStateToZooKeeper(context.getSessionZooKeeperClient(), applicationPackage, applicationId, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index bd130ebb153..fd32cb020d6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -128,7 +128,7 @@ public class SessionZooKeeperClient { public void delete(NestedTransaction transaction ) { try { - log.log(LogLevel.DEBUG, "Deleting " + sessionPath.getAbsolute()); + log.log(Level.FINE, "Deleting " + sessionPath.getAbsolute()); CuratorTransaction curatorTransaction = new CuratorTransaction(curator); CuratorOperations.deleteAll(sessionPath.getAbsolute(), curator).forEach(curatorTransaction::add); transaction.add(curatorTransaction); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index b82574efa9f..45b22ae156c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -121,10 +121,10 @@ public class TenantRepository { } else { this.directoryCache = Optional.empty(); } - log.log(LogLevel.DEBUG, "Creating all tenants"); + log.log(Level.FINE, "Creating all tenants"); bootstrapTenants(); notifyTenantsLoaded(); - log.log(LogLevel.DEBUG, "All tenants created"); + log.log(Level.FINE, "All tenants created"); checkForRemovedApplicationsService.scheduleWithFixedDelay(this::removeUnusedApplications, checkForRemovedApplicationsInterval.getSeconds(), checkForRemovedApplicationsInterval.getSeconds(), @@ -153,7 +153,7 @@ public class TenantRepository { /** Public for testing. */ public synchronized void updateTenants() { Set allTenants = readTenantsFromZooKeeper(curator); - log.log(LogLevel.DEBUG, "Create tenants, tenants found in zookeeper: " + allTenants); + log.log(Level.FINE, "Create tenants, tenants found in zookeeper: " + allTenants); for (TenantName tenantName : Set.copyOf(tenants.keySet())) if ( ! allTenants.contains(tenantName)) zkWatcherExecutor.execute(tenantName, () -> closeTenant(tenantName)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java index 6f0638a4760..b3d783633a5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java @@ -81,7 +81,7 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional vespaVersion) { Application application = getApplication(appId, vespaVersion); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); + log.log(Level.FINE, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); } return application.resolveConfig(req, responseFactory); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java index fdc96ea390e..6bf645cc02c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java @@ -27,7 +27,7 @@ public class InitializedCounter { } private void initializeCounterValue(Long latestSessionId) { - log.log(LogLevel.DEBUG, "path=" + sessionsDirPath + ", current=" + latestSessionId); + log.log(Level.FINE, "path=" + sessionsDirPath + ", current=" + latestSessionId); if (latestSessionId != null) { counter.initialize(latestSessionId); } else { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java index 76a934236d0..34cf6e345a0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java @@ -150,7 +150,7 @@ class ZKApplicationFile extends ApplicationFile { StringWriter writer = new StringWriter(); try { mapper.writeValue(writer, new MetaData(status, input == null ? "" : ConfigUtils.getMd5(input))); - log.log(LogLevel.DEBUG, "Writing meta file to " + metaPath); + log.log(Level.FINE, "Writing meta file to " + metaPath); zkApp.putData(metaPath, writer.toString()); } catch (IOException e) { throw new RuntimeException("Error writing meta file to " + metaPath, e); @@ -159,7 +159,7 @@ class ZKApplicationFile extends ApplicationFile { public MetaData getMetaData() { String metaPath = getZKPath(getMetaPath()); - log.log(LogLevel.DEBUG, "Getting metadata for " + metaPath); + log.log(Level.FINE, "Getting metadata for " + metaPath); if (!zkApp.exists(getZKPath(path))) { if (zkApp.exists(metaPath)) { return getMetaDataFromZk(metaPath); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java index 3ec515a1528..9e7576ae8f5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java @@ -42,7 +42,7 @@ public class DeployHandlerLoggerTest { } private void logMessages(DeployLogger logger) { - logger.log(LogLevel.DEBUG, "foobar"); + logger.log(Level.FINE, "foobar"); logger.log(LogLevel.SPAM, "foobar"); logger.log(LogLevel.FINE, "baz"); logger.log(LogLevel.WARNING, "baz"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index c5015536e03..a2e0b1e9443 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -131,7 +131,7 @@ public class SessionHandlerTest { status = Session.Status.PREPARE; this.dockerImageRepository = params.dockerImageRepository(); if (doVerboseLogging) { - logger.log(LogLevel.DEBUG, "debuglog"); + logger.log(Level.FINE, "debuglog"); } return actions; } diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index 1d4a174dc55..270decfeaeb 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -57,7 +57,7 @@ public class HandlersConfigurerDi { ComponentRegistry requestHandlerRegistry, ComponentRegistry clientProviderRegistry, ComponentRegistry serverProviderRegistry) { - log.log(LogLevel.DEBUG, "RegistriesHack.init " + System.identityHashCode(this)); + log.log(Level.FINE, "RegistriesHack.init " + System.identityHashCode(this)); vespaContainer.setComponentRegistry(allComponents); vespaContainer.setRequestHandlerRegistry(requestHandlerRegistry); diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index 8371acb59d4..6878b6df09d 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -228,7 +228,7 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { long endTime = System.currentTimeMillis(); writeToLogs(endTime); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Got exception when writing to client: " + Exceptions.toMessageString(throwable)); + log.log(Level.FINE, "Got exception when writing to client: " + Exceptions.toMessageString(throwable)); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 9e69b9890b5..2b4c1489313 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -70,7 +70,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler @Override public final void handleRequest(Request request, BufferedContentChannel requestContent, ResponseHandler responseHandler) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "In " + this.getClass() + ".handleRequest()"); + log.log(Level.FINE, "In " + this.getClass() + ".handleRequest()"); } com.yahoo.jdisc.http.HttpRequest jdiscRequest = asHttpRequest(request); HttpRequest httpRequest = new HttpRequest(jdiscRequest, new UnsafeContentInputStream(requestContent.toReadable())); @@ -192,7 +192,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler } catch (Exception e) { metric.add(RENDERING_ERRORS, 1, null); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Error writing response to client - connection probably terminated " + + log.log(Level.FINE, "Error writing response to client - connection probably terminated " + "from client side.", e); } return new DevNullChannel(); // Ignore further operations on this diff --git a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java index d76804a1d5f..699ebb7a705 100644 --- a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java @@ -39,7 +39,7 @@ public class JacksonJsonResponse extends HttpResponse { public void render(OutputStream outputStream) throws IOException { if (log.isLoggable(LogLevel.DEBUG)) { String json = jsonMapper.writeValueAsString(entity); - log.log(LogLevel.DEBUG, "Writing the following JSON to response output stream:\n" + json); + log.log(Level.FINE, "Writing the following JSON to response output stream:\n" + json); outputStream.write(json.getBytes()); } else { jsonMapper.writeValue(outputStream, entity); diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java index e0cb69d8499..fdaaf4b698d 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java @@ -255,7 +255,7 @@ public class ComponentGraph { if (component.isEmpty()) { Object instance; try { - log.log(LogLevel.DEBUG, "Trying the fallback injector to create" + messageForNoGlobalComponent(clazz, node)); + log.log(Level.FINE, "Trying the fallback injector to create" + messageForNoGlobalComponent(clazz, node)); instance = fallbackInjector.getInstance(key); } catch (ConfigurationException e) { throw removeStackTrace(new IllegalStateException( diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index ef9b4e8d1ca..c8cef145a91 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -128,7 +128,7 @@ public final class SessionCache extends AbstractComponent { } private static void logSystemInfo(ContainerMbusConfig containerMbusConfig, long maxPendingSize) { - log.log(LogLevel.DEBUG, + log.log(Level.FINE, "Running with maximum heap size of " + (Runtime.getRuntime().maxMemory() / 1024L / 1024L) + " MB"); log.log(LogLevel.CONFIG, "Amount of memory reserved for container core: " + containerMbusConfig.containerCoreMemory() + " MB."); @@ -204,7 +204,7 @@ public final class SessionCache extends AbstractComponent { @Override SharedSourceSession create(SourceSessionParams p) { - log.log(LogLevel.DEBUG, "Creating new source session."); + log.log(Level.FINE, "Creating new source session."); return messageBus.newSourceSession(p); } @@ -215,7 +215,7 @@ public final class SessionCache extends AbstractComponent { @Override void logReuse(final SharedSourceSession session) { - log.log(LogLevel.DEBUG, "Reusing source session."); + log.log(Level.FINE, "Reusing source session."); } } @@ -224,7 +224,7 @@ public final class SessionCache extends AbstractComponent { @Override SharedIntermediateSession create(IntermediateSessionParams p) { - log.log(LogLevel.DEBUG, "Creating new intermediate session " + p.getName() + ""); + log.log(Level.FINE, "Creating new intermediate session " + p.getName() + ""); return messageBus.newIntermediateSession(p); } @@ -235,7 +235,7 @@ public final class SessionCache extends AbstractComponent { @Override void logReuse(SharedIntermediateSession session) { - log.log(LogLevel.DEBUG, "Reusing intermediate session " + session.name() + ""); + log.log(Level.FINE, "Reusing intermediate session " + session.name() + ""); } } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 0966ae3e4c0..9e3f6d20e36 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -726,7 +726,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { if (includeQuery) message += ": [" + queryTreeText() + "]"; - log.log(LogLevel.DEBUG,message); + log.log(Level.FINE,message); // Pass 0 as traceLevel as the trace level check is already done above, // and it is not propagated to trace until execution has started diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index 30860f9570b..dbde0ad409f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -227,7 +227,7 @@ public class GroupingExecutor extends Searcher { } if (log.isLoggable(LogLevel.DEBUG)) { for (Grouping grouping : passList) { - log.log(LogLevel.DEBUG, "Pass(" + pass + "), Grouping(" + grouping.getId() + "): " + grouping); + log.log(Level.FINE, "Pass(" + pass + "), Grouping(" + grouping.getId() + "): " + grouping); } } Item passRoot; @@ -265,7 +265,7 @@ public class GroupingExecutor extends Searcher { } if (log.isLoggable(LogLevel.DEBUG)) { for (Grouping grouping : groupingMap.values()) { - log.log(LogLevel.DEBUG, "Result Grouping(" + grouping.getId() + "): " + grouping); + log.log(Level.FINE, "Result Grouping(" + grouping.getId() + "): " + grouping); } } return ret; diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index ee81f2c819e..2a8eeaf8e7f 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -154,7 +154,7 @@ public class SearchHandler extends LoggingRequestHandler { long numQueriesToTraceOnDebugAfterStartup, Optional hostResponseHeaderKey) { super(executor, accessLog, metric, true); - log.log(LogLevel.DEBUG, "SearchHandler.init " + System.identityHashCode(this)); + log.log(Level.FINE, "SearchHandler.init " + System.identityHashCode(this)); this.queryProfileRegistry = queryProfileRegistry; this.executionFactory = executionFactory; @@ -426,7 +426,7 @@ public class SearchHandler extends LoggingRequestHandler { } catch (ParseException e) { ErrorMessage error = ErrorMessage.createIllegalQuery("Could not parse query [" + request + "]: " + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, error::getDetailedMessage); + log.log(Level.FINE, error::getDetailedMessage); return new Result(query, error); } catch (IllegalArgumentException e) { if ("Comparison method violates its general contract!".equals(e.getMessage())) { @@ -438,7 +438,7 @@ public class SearchHandler extends LoggingRequestHandler { else { ErrorMessage error = ErrorMessage.createBadRequest("Invalid search request [" + request + "]: " + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, error::getDetailedMessage); + log.log(Level.FINE, error::getDetailedMessage); return new Result(query, error); } } catch (LinkageError | StackOverflowError e) { diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java index d4db21c3810..d66aff7bcd7 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java @@ -289,7 +289,7 @@ public class RewriterUtils { * @param msg Log message */ public static void log(Logger logger, String msg) { - logger.log(LogLevel.DEBUG, logger.getName() + ": " + msg); + logger.log(Level.FINE, logger.getName() + ": " + msg); } /** @@ -303,7 +303,7 @@ public class RewriterUtils { if(query!=null) { query.trace(logger.getName() + ": " + msg, true, TRACELEVEL); } - logger.log(LogLevel.DEBUG, logger.getName() + ": " + msg); + logger.log(Level.FINE, logger.getName() + ": " + msg); } /** diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java index 7d9d3f147f5..65b3fc5a805 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java @@ -52,7 +52,7 @@ public class InputCheckingSearcher extends Searcher { checkQuery(query); } catch (IllegalArgumentException e) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage()); + log.log(Level.FINE, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage()); } return new Result(query, ErrorMessage.createIllegalQuery(e.getMessage())); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index ad92512d1a1..f7c8c646bb4 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -330,20 +330,20 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { VisitorSession session = visitorSessionFactory.createVisitorSession(params); try { if ( !session.waitUntilDone(query.getTimeout())) { - log.log(LogLevel.DEBUG, "Visitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); + log.log(Level.FINE, "Visitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); session.abort(); throw new TimeoutException("Query timed out in " + VdsStreamingSearcher.class.getName()); } } finally { session.destroy(); sessionTrace = session.getTrace(); - log.log(LogLevel.DEBUG, () -> sessionTrace.toString()); + log.log(Level.FINE, () -> sessionTrace.toString()); query.trace(sessionTrace.toString(), false, 9); } if (params.getControlHandler().getResult().code == VisitorControlHandler.CompletionCode.SUCCESS) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "VdsVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); + log.log(Level.FINE, "VdsVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); } } else { throw new IllegalArgumentException("Query failed: " // TODO: Is it necessary to use a runtime exception? @@ -394,7 +394,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { final int hitCountTotal = sr.getTotalHitCount(); final int hitCount = sr.getHitCount(); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Got SearchResult with " + hitCountTotal + " in total and " + hitCount + " hits in real for query with selection " + params.getDocumentSelection()); + log.log(Level.FINE, "Got SearchResult with " + hitCountTotal + " in total and " + hitCount + " hits in real for query with selection " + params.getDocumentSelection()); } List newHits = new ArrayList<>(hitCount); @@ -449,7 +449,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private void handleSummary(DocumentSummary ds) { int summaryCount = ds.getSummaryCount(); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Got DocumentSummary with " + summaryCount + " summaries for query with selection " + params.getDocumentSelection()); + log.log(Level.FINE, "Got DocumentSummary with " + summaryCount + " summaries for query with selection " + params.getDocumentSelection()); } synchronized (summaryMap) { for (int i = 0; i < summaryCount; i++) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index c478c396ca3..ae9d35e6790 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -259,7 +259,7 @@ public class AthenzFacade implements AccessControl { } public List getDomainList(String prefix) { - log.log(LogLevel.DEBUG, String.format("getDomainList(prefix=%s)", prefix)); + log.log(Level.FINE, String.format("getDomainList(prefix=%s)", prefix)); return zmsClient.getDomainList(prefix); } @@ -279,7 +279,7 @@ public class AthenzFacade implements AccessControl { } private static void log(String format, Object... args) { - log.log(LogLevel.DEBUG, String.format(format, args)); + log.log(Level.FINE, String.format(format, args)); } private String resourceStringPrefix(AthenzDomain tenantDomain) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index b63b4e4fcc6..56d3b928b8f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -155,7 +155,7 @@ public class DeploymentTrigger { /** Attempts to trigger the given job. */ public void trigger(Job job) { - log.log(LogLevel.DEBUG, "Triggering " + job); + log.log(Level.FINE, "Triggering " + job); applications().lockApplicationOrThrow(TenantAndApplicationId.from(job.applicationId()), application -> { jobs.start(job.applicationId(), job.jobType, job.versions); applications().store(application.with(job.applicationId().instance(), instance -> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 5cfd07dcb2e..d10c4dd226b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -102,7 +102,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C errorBuilder.append("Talking to server ").append(uri.getHost()); errorBuilder.append(", got ").append(status).append(" ") .append(content).append("\n"); - log.log(LogLevel.DEBUG, () -> String.format("Got response from %s with status code %d and content:\n %s", + log.log(Level.FINE, () -> String.format("Got response from %s with status code %d and content:\n %s", uri.getHost(), status, content)); return Optional.empty(); } @@ -118,7 +118,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C } catch (Exception e) { errorBuilder.append("Talking to server ").append(uri.getHost()); errorBuilder.append(" got exception ").append(e.getMessage()); - log.log(LogLevel.DEBUG, e, () -> "Got exception while sending request to " + uri.getHost()); + log.log(Level.FINE, e, () -> "Got exception while sending request to " + uri.getHost()); return Optional.empty(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java index 2e7f75cbb9c..1f515035fe7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java @@ -59,7 +59,7 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase { }); } catch (Exception e) { - logger.log(LogLevel.DEBUG, () -> "Exception verifying signed request: " + Exceptions.toMessageString(e)); + logger.log(Level.FINE, () -> "Exception verifying signed request: " + Exceptions.toMessageString(e)); } return Optional.empty(); } diff --git a/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java index f074d4a287c..5036ce59727 100644 --- a/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java @@ -35,7 +35,7 @@ public class SimpleDocumentProcessor extends DocumentProcessor { */ public void process(DocumentPut put) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Ignored " + put); + log.log(Level.FINE, "Ignored " + put); } } @@ -48,7 +48,7 @@ public class SimpleDocumentProcessor extends DocumentProcessor { */ public void process(DocumentUpdate update) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Ignored " + update); + log.log(Level.FINE, "Ignored " + update); } } @@ -61,7 +61,7 @@ public class SimpleDocumentProcessor extends DocumentProcessor { */ public void process(DocumentRemove remove) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Ignored " + remove); + log.log(Level.FINE, "Ignored " + remove); } } @@ -97,7 +97,7 @@ public class SimpleDocumentProcessor extends DocumentProcessor { } } catch (RuntimeException e) { if (log.isLoggable(LogLevel.DEBUG) && initialSize != 1) { - log.log(LogLevel.DEBUG, + log.log(Level.FINE, "Processing of document failed, from processing.getDocumentOperations() containing " + initialSize + " DocumentOperation(s).", e); } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index caa8b0b282d..2ebc29a6b09 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -194,13 +194,13 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { private LaterTimerTask(DocumentProcessingTask processingTask, long delay) { this.delay = delay; - log.log(LogLevel.DEBUG, "Enqueueing in " + delay + " ms due to Progress.LATER: " + processingTask); + log.log(Level.FINE, "Enqueueing in " + delay + " ms due to Progress.LATER: " + processingTask); this.processingTask = processingTask; } @Override public void run() { - log.log(LogLevel.DEBUG, "Submitting after having waited " + delay + " ms in LATER queue: " + processingTask); + log.log(Level.FINE, "Submitting after having waited " + delay + " ms in LATER queue: " + processingTask); processingTask.submit(); } } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index 383918ce188..810e8c6ada4 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -203,7 +203,7 @@ public class DocumentProcessingTask implements Runnable { if (exception != null) { StringWriter backtrace = new StringWriter(); exception.printStackTrace(new PrintWriter(backtrace)); - log.log(LogLevel.DEBUG, "Failed to process " + processing + ": " + backtrace.toString()); + log.log(Level.FINE, "Failed to process " + processing + ": " + backtrace.toString()); } } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java index 47d639e5dc6..e8897f55a9e 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java @@ -92,7 +92,7 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { } } if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Forwarding " + messages.size() + " messages from " + processings.size() + + log.log(Level.FINE, "Forwarding " + messages.size() + " messages from " + processings.size() + " processings."); } if (messages.isEmpty()) { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java index 7c4a6c8c2c3..ca7455e1305 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java @@ -43,7 +43,7 @@ class MessageFactory { message.setTimeRemaining(requestMsg.getTimeRemainingNow()); message.getTrace().setLevel(requestMsg.getTrace().getLevel()); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Created '" + message.getClass().getName() + + log.log(Level.FINE, "Created '" + message.getClass().getName() + "', route = '" + message.getRoute() + "', priority = '" + message.getPriority().name() + "', load type = '" + message.getLoadType() + diff --git a/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java index 93d87a49c64..406fc9a6db9 100644 --- a/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java @@ -59,7 +59,7 @@ public class JoinerDocumentProcessor extends DocumentProcessor { if (op instanceof DocumentPut) { innerDocuments.add(((DocumentPut)op).getDocument()); } else { - log.log(LogLevel.DEBUG, "Skipping: " + op); + log.log(Level.FINE, "Skipping: " + op); } } processing.getDocumentOperations().clear(); diff --git a/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java index f6be7a4d990..7ec4ece3a3c 100644 --- a/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java @@ -42,7 +42,7 @@ public class SplitterDocumentProcessor extends DocumentProcessor { public Progress process(Processing processing) { if (processing.getDocumentOperations().size() != 1) { //we were given more than one document, return - log.log(LogLevel.DEBUG, "More than one document given, returning. (Was given " + log.log(Level.FINE, "More than one document given, returning. (Was given " + processing.getDocumentOperations().size() + " documents)."); return Progress.DONE; } @@ -57,14 +57,14 @@ public class SplitterDocumentProcessor extends DocumentProcessor { Array innerDocuments = (Array) outerDoc.getFieldValue(arrayFieldName); if (innerDocuments == null) { //the document does not have the field, return - log.log(LogLevel.DEBUG, "The given Document does not have a field value for field " + log.log(Level.FINE, "The given Document does not have a field value for field " + arrayFieldName + ", returning. (Was given " + outerDoc + ")."); return Progress.DONE; } if (innerDocuments.size() == 0) { //the array is empty, return - log.log(LogLevel.DEBUG, "The given Document does not have any elements in array field " + log.log(Level.FINE, "The given Document does not have any elements in array field " + arrayFieldName + ", returning. (Was given " + outerDoc + ")."); return Progress.DONE; } @@ -118,7 +118,7 @@ public class SplitterDocumentProcessor extends DocumentProcessor { static boolean doProcessOuterDocument(Object o, String documentTypeName) { if ( ! (o instanceof DocumentOperation)) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, o + " is not a DocumentOperation."); + log.log(Level.FINE, o + " is not a DocumentOperation."); } return false; } @@ -127,7 +127,7 @@ public class SplitterDocumentProcessor extends DocumentProcessor { if ( ! (outerDocOp instanceof DocumentPut)) { //this is not a put, return if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Given DocumentOperation is not a DocumentPut, returning. (Was given " + log.log(Level.FINE, "Given DocumentOperation is not a DocumentPut, returning. (Was given " + outerDocOp + ")."); } return false; @@ -138,7 +138,7 @@ public class SplitterDocumentProcessor extends DocumentProcessor { if (!type.getName().equalsIgnoreCase(documentTypeName)) { //this is not the right document type if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Given Document is of wrong type, returning. (Was given " + outerDoc + ")."); + log.log(Level.FINE, "Given Document is of wrong type, returning. (Was given " + outerDoc + ")."); } return false; } diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java index 30b7f9bf718..ec17e5e37e4 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java @@ -89,14 +89,14 @@ public class IndexingProcessor extends DocumentProcessor { private void processDocument(DocumentPut prev, List out) { DocumentScript script = scriptMgr.getScript(prev.getDocument().getDataType()); if (script == null) { - log.log(LogLevel.DEBUG, "No indexing script for document '%s'.", prev.getId()); + log.log(Level.FINE, "No indexing script for document '%s'.", prev.getId()); out.add(prev); return; } - log.log(LogLevel.DEBUG, "Processing document '%s'.", prev.getId()); + log.log(Level.FINE, "Processing document '%s'.", prev.getId()); Document next = script.execute(adapterFactory, prev.getDocument()); if (next == null) { - log.log(LogLevel.DEBUG, "Document '" + prev.getId() + "' produced no output."); + log.log(Level.FINE, "Document '" + prev.getId() + "' produced no output."); return; } @@ -106,14 +106,14 @@ public class IndexingProcessor extends DocumentProcessor { private void processUpdate(DocumentUpdate prev, List out) { DocumentScript script = scriptMgr.getScript(prev.getType()); if (script == null) { - log.log(LogLevel.DEBUG, "No indexing script for update '%s'.", prev.getId()); + log.log(Level.FINE, "No indexing script for update '%s'.", prev.getId()); out.add(prev); return; } - log.log(LogLevel.DEBUG, "Processing update '%s'.", prev.getId()); + log.log(Level.FINE, "Processing update '%s'.", prev.getId()); DocumentUpdate next = script.execute(adapterFactory, prev); if (next == null) { - log.log(LogLevel.DEBUG, "Update '" + prev.getId() + "' produced no output."); + log.log(Level.FINE, "Update '" + prev.getId() + "' produced no output."); return; } next.setCondition(prev.getCondition()); @@ -121,7 +121,7 @@ public class IndexingProcessor extends DocumentProcessor { } private void processRemove(DocumentRemove prev, List out) { - log.log(LogLevel.DEBUG, "Not processing remove '%s'.", prev.getId()); + log.log(Level.FINE, "Not processing remove '%s'.", prev.getId()); out.add(prev); } diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java index 783a003043e..a148966c250 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java @@ -35,22 +35,22 @@ public class ScriptManager { private Map getScripts(DocumentType inputType) { Map scripts = documentFieldScripts.get(inputType.getName()); if (scripts != null) { - log.log(LogLevel.DEBUG, "Using script for type '%s'.", inputType.getName()); + log.log(Level.FINE, "Using script for type '%s'.", inputType.getName()); return scripts; } for (Map.Entry> entry : documentFieldScripts.entrySet()) { if (inputType.inherits(docTypeMgr.getDocumentType(entry.getKey()))) { - log.log(LogLevel.DEBUG, "Using script of super-type '%s'.", entry.getKey()); + log.log(Level.FINE, "Using script of super-type '%s'.", entry.getKey()); return entry.getValue(); } } for (Map.Entry> entry : documentFieldScripts.entrySet()) { if (docTypeMgr.getDocumentType(entry.getKey()).inherits(inputType)) { - log.log(LogLevel.DEBUG, "Using script of sub-type '%s'.", entry.getKey()); + log.log(Level.FINE, "Using script of sub-type '%s'.", entry.getKey()); return entry.getValue(); } } - log.log(LogLevel.DEBUG, "No script for type '%s'.", inputType.getName()); + log.log(Level.FINE, "No script for type '%s'.", inputType.getName()); return null; } @@ -63,7 +63,7 @@ public class ScriptManager { if (fieldScripts != null) { DocumentScript script = fieldScripts.get(inputFieldName); if (script != null) { - log.log(LogLevel.DEBUG, "Using script for type '%s' and field '%s'.", inputType.getName(), inputFieldName); + log.log(Level.FINE, "Using script for type '%s' and field '%s'.", inputType.getName(), inputFieldName); return script; } } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index d5da805dcdc..1d81d9e6e78 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -72,7 +72,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub setupAnnotationTypesWithoutPayloads(config, manager); setupAnnotationRefTypes(config, manager); - log.log(LogLevel.DEBUG, "Configuring document manager with " + config.datatype().size() + " data types."); + log.log(Level.FINE, "Configuring document manager with " + config.datatype().size() + " data types."); ArrayList failed = new ArrayList<>(); failed.addAll(config.datatype()); while (!failed.isEmpty()) { @@ -226,7 +226,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub DocumentTypeManager manager = configureNewManager(config); int defaultTypeCount = new DocumentTypeManager().getDataTypes().size(); if (this.managerToConfigure.getDataTypes().size() != defaultTypeCount) { - log.log(LogLevel.DEBUG, "Live document config overwritten with new config."); + log.log(Level.FINE, "Live document config overwritten with new config."); } managerToConfigure.assign(manager); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java index 1842c1972df..eea3d6e66e8 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java @@ -306,7 +306,7 @@ public class ProgressToken { && !superbucket.contains(progress) && !progress.contains(superbucket)) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "updateProgress called with non-contained bucket " + log.log(Level.FINE, "updateProgress called with non-contained bucket " + "pair " + superbucket + ":" + progress + ", but allowing anyway"); } } @@ -323,7 +323,7 @@ public class ProgressToken { if (!progress.equals(FINISHED_BUCKET)) { if (entry.getState() != BucketState.BUCKET_ACTIVE) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "updateProgress called with sub-bucket that was " + log.log(Level.FINE, "updateProgress called with sub-bucket that was " + "not marked as active " + superbucket + ":" + progress); } } else { @@ -362,7 +362,7 @@ public class ProgressToken { protected void addBucket(BucketId superbucket, BucketId progress, BucketState state) { if (progress.equals(FINISHED_BUCKET)) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Trying to add already finished superbucket " + log.log(Level.FINE, "Trying to add already finished superbucket " + superbucket + "; ignoring it"); } return; @@ -796,7 +796,7 @@ public class ProgressToken { // be discarded if (rightSibling.getProgress().getUsedBits() != 0 && log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Bucket progress for " + rightCheck + + log.log(Level.FINE, "Bucket progress for " + rightCheck + " will be lost due to merging; potential for duplicates in result-set"); } buckets.remove(bucketToKeyWrapper(rightCheck)); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java index 83a7bc20353..ecf6d640ef7 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java @@ -111,7 +111,7 @@ public class VisitorIterator { if (!progress.isFinished()) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Importing unfinished progress token with " + + log.log(Level.FINE, "Importing unfinished progress token with " + "bits: " + progressToken.getDistributionBitCount() + ", active: " + progressToken.getActiveBucketCount() + ", pending: " + progressToken.getPendingBucketCount() + @@ -123,7 +123,7 @@ public class VisitorIterator { // Lower all active to pending if (progressToken.getActiveBucketCount() > 0) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Progress token had active buckets upon range " + + log.log(Level.FINE, "Progress token had active buckets upon range " + "construction. Setting these as pending"); } progressToken.setAllBucketsToState(ProgressToken.BucketState.BUCKET_PENDING); @@ -137,7 +137,7 @@ public class VisitorIterator { correctTruncatedBucketCursor(); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Partial bucket space progress; continuing "+ + log.log(Level.FINE, "Partial bucket space progress; continuing "+ "from position " + progressToken.getBucketCursor()); } } @@ -196,7 +196,7 @@ public class VisitorIterator { // single-bit bucket once did if (isLosslessResetPossible()) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "At start of bucket space and all " + + log.log(Level.FINE, "At start of bucket space and all " + "buckets have no progress; doing a lossless reset " + "instead of splitting/merging"); } @@ -263,7 +263,7 @@ public class VisitorIterator { } } if ((bucketsSplit > 0 || bucketsMerged > 0) && log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Existing progress' pending buckets had inconsistent " + + log.log(Level.FINE, "Existing progress' pending buckets had inconsistent " + "distribution bits; performed " + bucketsSplit + " split ops and " + bucketsMerged + " merge ops. Pending: " + pendingBefore + " -> " + p.getPendingBucketCount()); @@ -335,7 +335,7 @@ public class VisitorIterator { if (progressToken.getActiveBucketCount() > 0) { flushActive = true; if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Holding off new/pending buckets and consistency " + + log.log(Level.FINE, "Holding off new/pending buckets and consistency " + "correction until all " + progress.getActiveBucketCount() + " active buckets have been updated"); } @@ -350,7 +350,7 @@ public class VisitorIterator { correctInconsistentPending(distributionBitCount); if (delta > 0) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Increasing distribution bits for full bucket " + + log.log(Level.FINE, "Increasing distribution bits for full bucket " + "space range source from " + progressToken.getDistributionBitCount() + " to " + distributionBitCount); } @@ -360,7 +360,7 @@ public class VisitorIterator { progressToken.setBucketCursor(progressToken.getBucketCursor() << delta); } else if (delta < 0) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Decreasing distribution bits for full bucket " + + log.log(Level.FINE, "Decreasing distribution bits for full bucket " + "space range source from " + progressToken.getDistributionBitCount() + " to " + distributionBitCount + " bits"); } @@ -387,14 +387,14 @@ public class VisitorIterator { // consistency fix. This simplifies things greatly assert(flushActive); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received non-finished bucket " + + log.log(Level.FINE, "Received non-finished bucket " + superbucket + " with wrong distribution bit count (" + superbucket.getUsedBits() + "). Waiting to correct " + "until all active are done"); } } else { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received finished bucket " + + log.log(Level.FINE, "Received finished bucket " + superbucket + " with wrong distribution bit count (" + superbucket.getUsedBits() + "). Waiting to correct " + "until all active are done"); @@ -405,7 +405,7 @@ public class VisitorIterator { if (progressToken.getActiveBucketCount() == 0) { if (flushActive) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "All active buckets flushed, " + + log.log(Level.FINE, "All active buckets flushed, " + "correcting progress token and continuing normal operation"); } // Trigger the actual bucket state change this time @@ -509,7 +509,7 @@ public class VisitorIterator { progress.setDistributionBitCount(distributionBitCount); this.distributionBitCount = distributionBitCount; if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Set distribution bit count to " + log.log(Level.FINE, "Set distribution bit count to " + distributionBitCount + " for explicit bucket source (no-op)"); } } @@ -722,7 +722,7 @@ public class VisitorIterator { bucketSource.setDistributionBitCount(distBits, progressToken); distributionBitCount = distBits; if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Set visitor iterator distribution bit count to " + log.log(Level.FINE, "Set visitor iterator distribution bit count to " + distBits); } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java index 247364c8a7a..92abd8c98ca 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java @@ -56,7 +56,7 @@ public class MessageBusVisitorDestinationSession implements VisitorDestinationSe public void ack(AckToken token) { try { - log.log(LogLevel.DEBUG, "Sending ack " + token.ackObject); + log.log(Level.FINE, "Sending ack " + token.ackObject); session.reply((Reply) token.ackObject); } catch (Exception e) { e.printStackTrace(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index b6e822b597e..c2c36d01805 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -405,7 +405,7 @@ public class MessageBusVisitorSession implements VisitorSession { synchronized (progress.getToken()) { this.startTimeNanos = clock.monotonicNanoTime(); if (progress.getIterator().isDone()) { - log.log(LogLevel.DEBUG, sessionName + ": progress token indicates " + + log.log(Level.FINE, sessionName + ": progress token indicates " + "session is done before it could even start; no-op"); return; } @@ -433,7 +433,7 @@ public class MessageBusVisitorSession implements VisitorSession { * successful, will be equal to newState. */ private StateDescription transitionTo(StateDescription newState) { - log.log(LogLevel.DEBUG, sessionName + ": attempting transition to state " + newState); + log.log(Level.FINE, sessionName + ": attempting transition to state " + newState); switch (newState.getState()) { case WORKING: assert(state.getState() == State.NOT_STARTED); @@ -450,7 +450,7 @@ public class MessageBusVisitorSession implements VisitorSession { default: com.yahoo.protect.Process.logAndDie("Invalid target transition state: " + newState); } - log.log(LogLevel.DEBUG, "Session '" + sessionName + "' is now in state " + state); + log.log(Level.FINE, "Session '" + sessionName + "' is now in state " + state); return state; } @@ -500,7 +500,7 @@ public class MessageBusVisitorSession implements VisitorSession { // If no cluster route has been set by user arguments, attempt to retrieve it from mbus config. if (params.getRoute() == null || !params.getRoute().hasHops()) { params.setRoute(getClusterRoute(routingTable)); - log.log(LogLevel.DEBUG, "No route specified; resolved implicit " + + log.log(Level.FINE, "No route specified; resolved implicit " + "storage cluster: " + params.getRoute().toString()); } } @@ -575,7 +575,7 @@ public class MessageBusVisitorSession implements VisitorSession { progressToken); } else { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "parameters specify explicit bucket set " + + log.log(Level.FINE, "parameters specify explicit bucket set " + "to visit; using it rather than document selection (" + params.getBucketsToVisit().size() + " buckets given)"); } @@ -647,7 +647,7 @@ public class MessageBusVisitorSession implements VisitorSession { VisitorIterator.BucketProgress bucket = progress.getIterator().getNext(); Result result = sender.send(createMessage(bucket)); if (result.isAccepted()) { - log.log(LogLevel.DEBUG, sessionName + ": sent CreateVisitor for bucket " + + log.log(Level.FINE, sessionName + ": sent CreateVisitor for bucket " + bucket.getSuperbucket() + " with progress " + bucket.getProgress()); ++pendingMessageCount; } else { @@ -688,7 +688,7 @@ public class MessageBusVisitorSession implements VisitorSession { // 'done' is only ever written when token mutex is held, so safe to check // outside of completionMonitor lock. assert(!done) : "Session was marked as completed more than once"; - log.log(LogLevel.DEBUG, "Visitor session '" + sessionName + "' has completed"); + log.log(Level.FINE, "Visitor session '" + sessionName + "' has completed"); if (params.getLocalDataHandler() != null) { params.getLocalDataHandler().onDone(); } @@ -755,7 +755,7 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public void run() { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Visitor session " + sessionName + ": Received message " + message); + log.log(Level.FINE, "Visitor session " + sessionName + ": Received message " + message); } try { if (message instanceof VisitorInfoMessage) { @@ -810,7 +810,7 @@ public class MessageBusVisitorSession implements VisitorSession { msg.swapState(reply); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Visitor session " + sessionName + + log.log(Level.FINE, "Visitor session " + sessionName + ": Received VisitorInfo with " + msg.getFinishedBuckets().size() + " finished buckets"); } @@ -902,7 +902,7 @@ public class MessageBusVisitorSession implements VisitorSession { progress.getIterator().update(bucket, subProgress); String message = getErrorMessage(reply.getError(0)); - log.log(LogLevel.DEBUG, sessionName + ": received error reply for bucket " + + log.log(Level.FINE, sessionName + ": received error reply for bucket " + bucket + " with message '" + message + "'"); if (isFatalError(reply)) { @@ -1027,7 +1027,7 @@ public class MessageBusVisitorSession implements VisitorSession { BucketId superbucket = msg.getBuckets().get(0); BucketId subBucketProgress = reply.getLastBucket(); - log.log(LogLevel.DEBUG, sessionName + ": received CreateVisitorReply for bucket " + + log.log(Level.FINE, sessionName + ": received CreateVisitorReply for bucket " + superbucket + " with progress " + subBucketProgress); progress.getIterator().update(superbucket, subBucketProgress); @@ -1045,7 +1045,7 @@ public class MessageBusVisitorSession implements VisitorSession { int newMaxBuckets = Math.max(Math.min((int)(params.getMaxBucketsPerVisitor() * params.getDynamicMaxBucketsIncreaseFactor()), 128), 1); params.setMaxBucketsPerVisitor(newMaxBuckets); - log.log(LogLevel.DEBUG, sessionName + ": increasing max buckets per visitor to " + log.log(Level.FINE, sessionName + ": increasing max buckets per visitor to " + params.getMaxBucketsPerVisitor()); } } @@ -1055,7 +1055,7 @@ public class MessageBusVisitorSession implements VisitorSession { ClusterState newState = new ClusterState(reply.getSystemState()); int stateBits = newState.getDistributionBitCount(); if (stateBits != progress.getIterator().getDistributionBitCount()) { - log.log(LogLevel.DEBUG, "System state changed; now at " + + log.log(Level.FINE, "System state changed; now at " + stateBits + " distribution bits"); // Update the internal state of the visitor iterator. If we're increasing // the number of distribution bits, this may lead to splitting of pending @@ -1102,7 +1102,7 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public void ack(AckToken token) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Visitor session " + sessionName + + log.log(Level.FINE, "Visitor session " + sessionName + ": Sending ack " + token.ackObject); } // No locking here; replying should be thread safe in itself @@ -1144,7 +1144,7 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public void destroy() { - log.log(LogLevel.DEBUG, sessionName + ": synchronous destroy() called"); + log.log(Level.FINE, sessionName + ": synchronous destroy() called"); try { synchronized (progress.getToken()) { synchronized (completionMonitor) { @@ -1172,7 +1172,7 @@ public class MessageBusVisitorSession implements VisitorSession { } catch (Exception e) { log.log(LogLevel.ERROR, "Caught exception destroying communication interfaces", e); } - log.log(LogLevel.DEBUG, sessionName + ": synchronous destroy() done"); + log.log(Level.FINE, sessionName + ": synchronous destroy() done"); } } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java index 236a2a6c0f6..f89ab28ee16 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java @@ -395,7 +395,7 @@ public class StoragePolicy extends SlobrokPolicy { return targetSpec; } else { sendRandomReason = "Want to use distributor " + target + " but it is not in slobrok. Sending to random."; - log.log(LogLevel.DEBUG, "Target distributor is not in slobrok"); + log.log(Level.FINE, "Target distributor is not in slobrok"); } } else { context.setContext(new MessageContext(cachedClusterState)); @@ -407,7 +407,7 @@ public class StoragePolicy extends SlobrokPolicy { context.setReply(reply); return null; } catch (Distribution.NoDistributorsAvailableException e) { - log.log(LogLevel.DEBUG, "No distributors available; clearing cluster state"); + log.log(Level.FINE, "No distributors available; clearing cluster state"); safeCachedClusterState.set(null); sendRandomReason = "No distributors available. Sending to random distributor."; context.setContext(createRandomDistributorTargetContext()); @@ -482,7 +482,7 @@ public class StoragePolicy extends SlobrokPolicy { // reject an operation bound to a particular bucket if it does not own the bucket in _both_ // the current and the next (transition target) state. Since it can happen during normal operation // and will happen per client operation, we keep this as debug level to prevent spamming the logs. - log.log(LogLevel.DEBUG, msg); + log.log(Level.FINE, msg); } else if (newState.getVersion() > context.usedState.getVersion()) { if (reply.getTrace().shouldTrace(1)) { reply.getTrace().trace(1, "Message sent to distributor " + context.calculatedDistributor + diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java index e83a78df719..e95eaa754b6 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java @@ -60,7 +60,7 @@ class FileAcquirerImpl implements FileAcquirer { logWarning(); target.close(); } else { - log.log(LogLevel.DEBUG, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + log.log(Level.FINE, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); pauseTime = 0; logCount = 0; return; @@ -152,7 +152,7 @@ class FileAcquirerImpl implements FileAcquirer { request.parameters().add(new StringValue(fileReference.value())); double rpcTimeout = Math.min(timer.timeLeft(TimeUnit.SECONDS), 60.0); - log.log(LogLevel.DEBUG, "InvokeSync waitFor " + fileReference + " with " + rpcTimeout + " seconds timeout"); + log.log(Level.FINE, "InvokeSync waitFor " + fileReference + " with " + rpcTimeout + " seconds timeout"); target.invokeSync(request, rpcTimeout); if (request.checkReturnTypes("s")) { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java index 5ea4092e0c6..66b713ac6ba 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java @@ -63,7 +63,7 @@ public class CompressedFileReference { } static void decompress(File inputFile, File outputDir) throws IOException { - log.log(LogLevel.DEBUG, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); + log.log(Level.FINE, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); try (ArchiveInputStream ais = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(inputFile)))) { decompress(ais, outputDir); } catch (IllegalArgumentException e) { @@ -78,7 +78,7 @@ public class CompressedFileReference { File outFile = new File(outputFile, entry.getName()); if (entry.isDirectory()) { if (!(outFile.exists() && outFile.isDirectory())) { - log.log(LogLevel.DEBUG, () -> "Creating dir: " + outFile.getAbsolutePath()); + log.log(Level.FINE, () -> "Creating dir: " + outFile.getAbsolutePath()); if (!outFile.mkdirs()) { log.log(LogLevel.WARNING, "Could not create dir " + entry.getName()); } @@ -113,7 +113,7 @@ public class CompressedFileReference { } private static void writeFileToTar(ArchiveOutputStream taos, File baseDir, File file) throws IOException { - log.log(LogLevel.DEBUG, () -> "Adding file to tar: " + baseDir.toPath().relativize(file.toPath()).toString()); + log.log(Level.FINE, () -> "Adding file to tar: " + baseDir.toPath().relativize(file.toPath()).toString()); taos.putArchiveEntry(taos.createArchiveEntry(file, baseDir.toPath().relativize(file.toPath()).toString())); ByteStreams.copy(new FileInputStream(file), taos); taos.closeArchiveEntry(); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 57281d4a84b..cb3a3caf2d4 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -89,7 +89,7 @@ public class FileDownloader { } else if (!file.canRead()) { throw new RuntimeException("File reference '" + fileReference.value() + "'exists, but unable to read it"); } else { - log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' found: " + file.getAbsolutePath()); + log.log(Level.FINE, () -> "File reference '" + fileReference.value() + "' found: " + file.getAbsolutePath()); fileReferenceDownloader.setDownloadStatus(fileReference, 1.0); return Optional.of(file); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 0228ad0a5a2..a79f3831c17 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -123,7 +123,7 @@ public class FileReceiver { log.log(LogLevel.ERROR, "Failed creating directory (" + fileReferenceDir.toPath() + "): " + e.getMessage(), e); throw new RuntimeException("Failed creating directory (" + fileReferenceDir.toPath() + "): ", e); } - log.log(LogLevel.DEBUG, () -> "Uncompressed file, moving to " + file.getAbsolutePath()); + log.log(Level.FINE, () -> "Uncompressed file, moving to " + file.getAbsolutePath()); moveFileToDestination(inprogressFile, file); } } catch (IOException e) { @@ -188,7 +188,7 @@ public class FileReceiver { private static void moveFileToDestination(File tempFile, File destination) { try { Files.move(tempFile.toPath(), destination.toPath()); - log.log(LogLevel.DEBUG, () -> "File moved from " + tempFile.getAbsolutePath()+ " to " + destination.getAbsolutePath()); + log.log(Level.FINE, () -> "File moved from " + tempFile.getAbsolutePath()+ " to " + destination.getAbsolutePath()); } catch (FileAlreadyExistsException e) { // Don't fail if it already exists (we might get the file from several config servers when retrying, servers are down etc. // so it might be written already). Delete temp file/dir in that case, to avoid filling the disk. @@ -215,7 +215,7 @@ public class FileReceiver { } private void receiveFileMeta(Request req) { - log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); String fileName = req.parameters().get(1).asString(); String type = req.parameters().get(2).asString(); @@ -240,7 +240,7 @@ public class FileReceiver { } private void receiveFilePart(Request req) { - log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); int sessionId = req.parameters().get(1).asInt32(); @@ -261,7 +261,7 @@ public class FileReceiver { } private void receiveFileEof(Request req) { - log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); int sessionId = req.parameters().get(1).asInt32(); long xxhash = req.parameters().get(2).asInt64(); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 4a3c474a436..35f8636c0f9 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -81,7 +81,7 @@ public class FileReferenceDownloader { void addToDownloadQueue(FileReferenceDownload fileReferenceDownload) { FileReference fileReference = fileReferenceDownload.fileReference(); - log.log(LogLevel.DEBUG, () -> "Will download file reference '" + fileReference.value() + "' with timeout " + downloadTimeout); + log.log(Level.FINE, () -> "Will download file reference '" + fileReference.value() + "' with timeout " + downloadTimeout); synchronized (downloads) { downloads.put(fileReference, fileReferenceDownload); downloadStatus.put(fileReference, 0.0); @@ -97,7 +97,7 @@ public class FileReferenceDownloader { downloads.remove(fileReference); download.future().complete(Optional.of(file)); } else { - log.log(LogLevel.DEBUG, () -> "Received '" + fileReference + "', which was not requested. Can be ignored if happening during upgrades/restarts"); + log.log(Level.FINE, () -> "Received '" + fileReference + "', which was not requested. Can be ignored if happening during upgrades/restarts"); } } } diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java index 0f69003d7ee..18cd6cf02c1 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java @@ -79,7 +79,7 @@ public class AthenzAuthorizationFilter extends JsonSecurityRequestFilterBase { try { Optional resourceMapping = requestResourceMapper.getResourceNameAndAction(request.getMethod(), request.getRequestURI(), request.getQueryString()); - log.log(LogLevel.DEBUG, () -> String.format("Resource mapping for '%s': %s", request, resourceMapping)); + log.log(Level.FINE, () -> String.format("Resource mapping for '%s': %s", request, resourceMapping)); if (resourceMapping.isEmpty()) { incrementAcceptedMetrics(request, false); return Optional.empty(); @@ -92,11 +92,11 @@ public class AthenzAuthorizationFilter extends JsonSecurityRequestFilterBase { incrementAcceptedMetrics(request, true); return Optional.empty(); } - log.log(LogLevel.DEBUG, () -> String.format("Forbidden (403) for '%s': %s", request, resultType.name())); + log.log(Level.FINE, () -> String.format("Forbidden (403) for '%s': %s", request, resultType.name())); incrementRejectedMetrics(request, FORBIDDEN, resultType.name()); return Optional.of(new ErrorResponse(FORBIDDEN, "Access forbidden: " + resultType.getDescription())); } catch (IllegalArgumentException e) { - log.log(LogLevel.DEBUG, () -> String.format("Unauthorized (401) for '%s': %s", request, e.getMessage())); + log.log(Level.FINE, () -> String.format("Unauthorized (401) for '%s': %s", request, e.getMessage())); incrementRejectedMetrics(request, UNAUTHORIZED, "Unauthorized"); return Optional.of(new ErrorResponse(UNAUTHORIZED, e.getMessage())); } @@ -196,7 +196,7 @@ public class AthenzAuthorizationFilter extends JsonSecurityRequestFilterBase { } private static void setAttribute(DiscFilterRequest request, String name, String value) { - log.log(LogLevel.DEBUG, () -> String.format("Setting attribute on '%s': '%s' = '%s'", request, name, value)); + log.log(Level.FINE, () -> String.format("Setting attribute on '%s': '%s' = '%s'", request, name, value)); request.setAttribute(name, value); } diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java index 514fdbcc745..ec6b1361ebc 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java @@ -41,7 +41,7 @@ public abstract class JsonSecurityRequestFilterBase extends AbstractComponent im errorMessage.put("message", error.message); error.response.headers().put("Content-Type", "application/json"); // Note: Overwrites header if already exists error.response.headers().put("Cache-Control", "must-revalidate,no-cache,no-store"); - log.log(LogLevel.DEBUG, () -> String.format("Error response for '%s': statusCode=%d, errorCode=%d, message='%s'", + log.log(Level.FINE, () -> String.format("Error response for '%s': statusCode=%d, errorCode=%d, message='%s'", request, error.response.getStatus(), error.errorCode, error.message)); try (FastContentWriter writer = ResponseDispatch.newInstance(error.response).connectFastWriter(responseHandler)) { writer.write(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(errorMessage)); diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java index 6ecb36f7af9..8045e35604a 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java @@ -72,7 +72,7 @@ public final class MbusClient extends AbstractResource implements ClientProvider @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying message bus client."); + log.log(Level.FINE, "Destroying message bus client."); sessionReference.close(); done = true; } diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java index 5893e4f6252..e26e1e7e134 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java @@ -42,19 +42,19 @@ public final class MbusServer extends AbstractResource implements ServerProvider @Override public void start() { - log.log(LogLevel.DEBUG, "Starting message bus server."); + log.log(Level.FINE, "Starting message bus server."); running.set(true); } @Override public void close() { - log.log(LogLevel.DEBUG, "Closing message bus server."); + log.log(Level.FINE, "Closing message bus server."); running.set(false); } @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying message bus server."); + log.log(Level.FINE, "Destroying message bus server."); running.set(false); sessionReference.close(); } diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java index d9db66b6041..7da164757cd 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java @@ -78,7 +78,7 @@ public class SharedDestinationSession extends AbstractResource implements Messag @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying shared destination session."); + log.log(Level.FINE, "Destroying shared destination session."); session.destroy(); mbusReference.close(); } diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java index 9a50101a492..5c9fab46e34 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java @@ -97,7 +97,7 @@ public class SharedIntermediateSession extends AbstractResource @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying shared intermediate session."); + log.log(Level.FINE, "Destroying shared intermediate session."); session.destroy(); mbusReference.close(); } diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java index 8c084f0f9e4..dd135a51378 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java @@ -35,7 +35,7 @@ public class SharedMessageBus extends AbstractResource { @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying shared message bus."); + log.log(Level.FINE, "Destroying shared message bus."); mbus.destroy(); } diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java index a75bf6513f9..56071682349 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java @@ -50,7 +50,7 @@ public class SharedSourceSession extends AbstractResource implements ClientSessi @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying shared source session."); + log.log(Level.FINE, "Destroying shared source session."); session.close(); mbusReference.close(); } diff --git a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java index b1d8de1bd98..580c621cb20 100644 --- a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java @@ -40,7 +40,7 @@ public abstract class AbstractPluginLoader implements PluginLoader { try { plugin.initPlugin(config); - log.log(LogLevel.DEBUG, pname + ": plugin loaded"); + log.log(Level.FINE, pname + ": plugin loaded"); } catch (Exception e) { log.log(LogLevel.ERROR, pname + ": init failed", e); } diff --git a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java index c67dbe3bce8..3a87ec920a8 100644 --- a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java @@ -17,12 +17,12 @@ public class BuiltinPluginLoader extends AbstractPluginLoader { private static final Logger log = Logger.getLogger(BuiltinPluginLoader.class.getName()); public void loadPlugins() { - log.log(LogLevel.DEBUG, "starting to load builtin plugins"); + log.log(Level.FINE, "starting to load builtin plugins"); loadFromClass(ArchiverPlugin.class); loadFromClass(LogMetricsPlugin.class); - log.log(LogLevel.DEBUG, "done loading builtin plugins"); + log.log(Level.FINE, "done loading builtin plugins"); } } diff --git a/logserver/src/main/java/com/yahoo/logserver/Flusher.java b/logserver/src/main/java/com/yahoo/logserver/Flusher.java index 8958d350a89..b155c8eb893 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Flusher.java +++ b/logserver/src/main/java/com/yahoo/logserver/Flusher.java @@ -46,7 +46,7 @@ public class Flusher extends Thread { h.flush(); } if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Flushing " + h); + log.log(Level.FINE, "Flushing " + h); } } } diff --git a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java index e74fa7a81f0..7c34963ae4f 100644 --- a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java +++ b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java @@ -118,7 +118,7 @@ public class LogDispatcher implements LogHandler { for (LogHandler h : handlers) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Flushing " + h.toString()); + log.log(Level.FINE, "Flushing " + h.toString()); } h.flush(); } @@ -138,7 +138,7 @@ public class LogDispatcher implements LogHandler { } handlers.clear(); - log.log(LogLevel.DEBUG, "Logdispatcher shut down. Handled " + messageCount + " messages"); + log.log(Level.FINE, "Logdispatcher shut down. Handled " + messageCount + " messages"); } /** diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index e73ae531ecc..ea6883cbe29 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -212,12 +212,12 @@ public class ArchiverHandler extends AbstractLogHandler { // ensure that root dir exists if (root.isDirectory()) { - log.log(LogLevel.DEBUG, "Using " + absoluteRootDir + " as root"); + log.log(Level.FINE, "Using " + absoluteRootDir + " as root"); } else { if (! root.mkdirs()) { log.log(LogLevel.ERROR, "Unable to create directory " + absoluteRootDir); } else { - log.log(LogLevel.DEBUG, "Created root at " + absoluteRootDir); + log.log(Level.FINE, "Created root at " + absoluteRootDir); } } filesArchived = new FilesArchived(root); diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java index 8635534ba7b..9d550a16a7e 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java @@ -66,7 +66,7 @@ public class LogWriter { // if file does not exist we have a winner if (! f.exists()) { - log.log(LogLevel.DEBUG, "nextWriter, new file: " + name); + log.log(Level.FINE, "nextWriter, new file: " + name); currentFile = f; bytesWritten = 0; return new FileWriter(f, true); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java index d2e3c84baab..db464c1cd99 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java @@ -101,7 +101,7 @@ public class DynamicThrottlePolicy extends StaticThrottlePolicy { localMaxThroughput = throughput; windowSize += weight*windowSizeIncrement; if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput); + log.log(Level.FINE, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput); } } else { // scale up/down throughput for comparing to window size @@ -120,7 +120,7 @@ public class DynamicThrottlePolicy extends StaticThrottlePolicy { windowSize += weight*windowSizeIncrement; } if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput + " efficiency " + efficiency); + log.log(Level.FINE, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput + " efficiency " + efficiency); } } windowSize = Math.max(minWindowSize, windowSize); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java b/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java index da143877083..77bbd662cc4 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java @@ -59,7 +59,7 @@ public class SendProxy implements MessageHandler, ReplyHandler { Trace trace = msg.getTrace(); if (logTrace) { if (reply.hasErrors()) { - log.log(LogLevel.DEBUG, "Trace for reply with error(s):\n" + reply.getTrace()); + log.log(Level.FINE, "Trace for reply with error(s):\n" + reply.getTrace()); } else if (log.isLoggable(LogLevel.SPAM)) { log.log(LogLevel.SPAM, "Trace for reply:\n" + reply.getTrace()); } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java index 9cab213b628..b8faafe54bb 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java @@ -102,12 +102,12 @@ public class RPCTarget implements RequestWaiter { synchronized (this) { if (version != null) { if (shouldLog) { - log.log(LogLevel.DEBUG, "Version already available for target '" + name + "' (version " + version + ")."); + log.log(Level.FINE, "Version already available for target '" + name + "' (version " + version + ")."); } hasVersion = true; } else { if (shouldLog) { - log.log(LogLevel.DEBUG, "Registering version handler '" + handler + "' for target '" + name + "'."); + log.log(Level.FINE, "Registering version handler '" + handler + "' for target '" + name + "'."); } versionHandlers.add(handler); if (!targetInvoked) { @@ -120,7 +120,7 @@ public class RPCTarget implements RequestWaiter { handler.handleVersion(version); } else if (shouldInvoke) { if (shouldLog) { - log.log(LogLevel.DEBUG, "Invoking mbus.getVersion() on target '" + name + "'"); + log.log(Level.FINE, "Invoking mbus.getVersion() on target '" + name + "'"); } Request req = new Request("mbus.getVersion"); target.invokeAsync(req, timeout, this); @@ -138,7 +138,7 @@ public class RPCTarget implements RequestWaiter { try { version = new Version(str); if (shouldLog) { - log.log(LogLevel.DEBUG, "Target '" + name + "' has version " + version + "."); + log.log(Level.FINE, "Target '" + name + "' has version " + version + "."); } } catch (IllegalArgumentException e) { log.log(LogLevel.WARNING, "Failed to parse '" + str + "' as version for target '" + name + "'.", e); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index 8edae823a97..4d57e89fc76 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -69,7 +69,7 @@ public class ApplicationMetricsRetriever extends AbstractComponent { } public Map> getMetrics(ConsumerId consumer) { - log.log(LogLevel.DEBUG, () -> "Retrieving metrics from " + clients.size() + " nodes."); + log.log(Level.FINE, () -> "Retrieving metrics from " + clients.size() + " nodes."); var forkJoinTask = forkJoinPool.submit(() -> clients.parallelStream() .map(client -> getNodeMetrics(client, consumer)) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))); @@ -77,7 +77,7 @@ public class ApplicationMetricsRetriever extends AbstractComponent { try { var metricsByNode = forkJoinTask.get(taskTimeout.toMillis(), TimeUnit.MILLISECONDS); - log.log(LogLevel.DEBUG, () -> "Finished retrieving metrics from " + clients.size() + " nodes."); + log.log(Level.FINE, () -> "Finished retrieving metrics from " + clients.size() + " nodes."); return metricsByNode; } catch (Exception e) { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java index 449309d8a3e..06036aa90d5 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java @@ -58,7 +58,7 @@ public class ConfigSentinelClient extends AbstractComponent { synchronized void ping(VespaService s) { List services = new ArrayList<>(); services.add(s); - log.log(LogLevel.DEBUG, "Ping for service " + s); + log.log(Level.FINE, "Ping for service " + s); try { setStatus(services); } catch (Exception e) { @@ -92,7 +92,7 @@ public class ConfigSentinelClient extends AbstractComponent { //from the sentinel for (VespaService s : services) { if ((!s.getServiceName().equals("configserver")) && !updatedServices.contains(s)) { - log.log(LogLevel.DEBUG,"Service " + s + " is no longer found with sentinel - setting alive = false"); + log.log(Level.FINE,"Service " + s + " is no longer found with sentinel - setting alive = false"); s.setAlive(false); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index a5c0e0e2395..0a17810d3cc 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -40,11 +40,11 @@ public abstract class HttpMetricFetcher { String u = "http://localhost:" + port + path; this.url = URI.create(u); - log.log(LogLevel.DEBUG, "Fetching metrics from " + u + " with timeout " + CONNECTION_TIMEOUT); + log.log(Level.FINE, "Fetching metrics from " + u + " with timeout " + CONNECTION_TIMEOUT); } String getJson() throws IOException { - log.log(LogLevel.DEBUG, "Connecting to url " + url + " for service '" + service + "'"); + log.log(Level.FINE, "Connecting to url " + url + " for service '" + service + "'"); return httpClient.execute(new HttpGet(url), new BasicResponseHandler()); } @@ -66,7 +66,7 @@ public abstract class HttpMetricFetcher { if (service.isAlive() && timesFetched > 5) { log.log(LogLevel.INFO, message); } else { - log.log(LogLevel.DEBUG, message); + log.log(Level.FINE, message); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java index 4bc9055c6a5..1cab1a859a9 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java @@ -64,7 +64,7 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { return HealthMetric.get(code, message); } catch (JSONException e) { - log.log(LogLevel.DEBUG, "Failed to parse json response from metrics page:" + e + ":" + data); + log.log(Level.FINE, "Failed to parse json response from metrics page:" + e + ":" + data); return HealthMetric.getUnknown("Not able to parse json from status page"); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java index 3e8eeea1268..7620e94fc9e 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java @@ -81,7 +81,7 @@ public class SystemPoller { br.close(); } catch (IOException ex) { - log.log(LogLevel.DEBUG, "Unable to read line from smaps file", ex); + log.log(Level.FINE, "Unable to read line from smaps file", ex); return size; } @@ -101,7 +101,7 @@ public class SystemPoller { return; } - log.log(LogLevel.DEBUG, "Monitoring system metrics for " + services.size() + " services"); + log.log(Level.FINE, "Monitoring system metrics for " + services.size() + " services"); long sysJiffies = getNormalizedSystemJiffies(); for (VespaService s : services) { @@ -112,10 +112,10 @@ public class SystemPoller { } Metrics metrics = new Metrics(); - log.log(LogLevel.DEBUG, "Current size of system metrics for service " + s + " is " + metrics.size()); + log.log(Level.FINE, "Current size of system metrics for service " + s + " is " + metrics.size()); long[] size = getMemoryUsage(s); - log.log(LogLevel.DEBUG, "Updating memory metric for service " + s); + log.log(Level.FINE, "Updating memory metric for service " + s); metrics.add(new Metric("memory_virt", size[memoryTypeVirtual], startTime / 1000)); metrics.add(new Metric("memory_rss", size[memoryTypeResident], startTime / 1000)); @@ -152,7 +152,7 @@ public class SystemPoller { try { in = new BufferedReader(new FileReader("/proc/" + pid + "/stat")); } catch (FileNotFoundException ex) { - log.log(LogLevel.DEBUG, "Unable to find pid " + pid + " in proc directory, for service " + service.getInstanceName()); + log.log(Level.FINE, "Unable to find pid " + pid + " in proc directory, for service " + service.getInstanceName()); service.setAlive(false); return 0; } @@ -161,7 +161,7 @@ public class SystemPoller { line = in.readLine(); in.close(); } catch (IOException ex) { - log.log(LogLevel.DEBUG, "Unable to read line from process stat file", ex); + log.log(Level.FINE, "Unable to read line from process stat file", ex); return 0; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java index 25e2b9068ab..5c7d8b3ce83 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java @@ -76,7 +76,7 @@ public class Telegraf extends AbstractComponent { ProcessResult processResult = uncheck(() -> processExecutor.execute(command)) .orElseThrow(() -> new RuntimeException("Timed out running command: " + command)); - logger.log(LogLevel.DEBUG, () -> String.format("Exit code: %d\nstdOut: %s\nstdErr: %s", + logger.log(Level.FINE, () -> String.format("Exit code: %d\nstdOut: %s\nstdErr: %s", processResult.exitCode, processResult.stdOut, processResult.stdErr)); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java index 0f36bf6125e..3864d25c2a6 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java @@ -18,12 +18,12 @@ public class TelegrafRegistry { private static final Logger logger = Logger.getLogger(TelegrafRegistry.class.getName()); public void addInstance(Telegraf telegraf) { - logger.log(LogLevel.DEBUG, () -> "Adding Telegraf instance to registry: " + telegraf.hashCode()); + logger.log(Level.FINE, () -> "Adding Telegraf instance to registry: " + telegraf.hashCode()); telegrafInstances.add(telegraf); } public void removeInstance(Telegraf telegraf) { - logger.log(LogLevel.DEBUG, () -> "Removing Telegraf instance from registry: " + telegraf.hashCode()); + logger.log(Level.FINE, () -> "Removing Telegraf instance from registry: " + telegraf.hashCode()); telegrafInstances.remove(telegraf); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java index 66927e6a2b9..3e2c14a2935 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java @@ -17,7 +17,7 @@ public class ProcessStarterImpl implements ProcessStarter { @Override public ProcessApi2 start(ProcessBuilder processBuilder) { if (logger.isLoggable(LogLevel.DEBUG)) { - logger.log(LogLevel.DEBUG, "Spawning process: " + processBuilder.command()); + logger.log(Level.FINE, "Spawning process: " + processBuilder.command()); } Process process = uncheck(processBuilder::start); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java index aef6cce2a46..6d2c074ab36 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java @@ -52,7 +52,7 @@ public class DnsNameResolver implements NameResolver { return InetAddresses.isInetAddress(hostname) ? Optional.empty() : Optional.of(hostname); } catch (UnknownHostException e) { // This is not an exceptional state hence the debug level - logger.log(LogLevel.DEBUG, "Unable to resolve ipaddress", e); + logger.log(Level.FINE, "Unable to resolve ipaddress", e); } return Optional.empty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index df3d2dedb4e..8a31b271406 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -101,7 +101,7 @@ public class InfraDeployerImpl implements InfraDeployer { prepare(); if (hostSpecs.isEmpty()) { - logger.log(LogLevel.DEBUG, "No nodes to provision for " + application.getCapacity().type() + ", removing application"); + logger.log(Level.FINE, "No nodes to provision for " + application.getCapacity().type() + ", removing application"); removeApplication(application.getApplicationId()); } else { NestedTransaction nestedTransaction = new NestedTransaction(); @@ -112,7 +112,7 @@ public class InfraDeployerImpl implements InfraDeployer { application.getApplicationId(), hostSpecs.stream().map(HostSpec::hostname).map(HostName::from).collect(Collectors.toList())); - logger.log(LogLevel.DEBUG, () -> generateActivationLogMessage(hostSpecs, application.getApplicationId())); + logger.log(Level.FINE, () -> generateActivationLogMessage(hostSpecs, application.getApplicationId())); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index d48f02e18e8..3532d989d85 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -166,7 +166,7 @@ public class LoadBalancerProvisioner { reals.add(new Real(HostName.from(node.hostname()), ip)); } } - log.log(LogLevel.DEBUG, "Creating load balancer for " + cluster + " in " + application.toShortString() + + log.log(Level.FINE, "Creating load balancer for " + cluster + " in " + application.toShortString() + ", targeting: " + reals); try { return service.create(application, cluster, reals, force); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index e8d8cd209f5..590847b8b7e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -149,7 +149,7 @@ public class NodePrioritizer { if (!hostHasCapacityForWantedFlavor || conflictingCluster) continue; - log.log(LogLevel.DEBUG, "Trying to add new Docker node on " + host); + log.log(Level.FINE, "Trying to add new Docker node on " + host); Optional allocation; try { allocation = host.ipConfig().pool().findAllocation(allNodes, nameResolver); @@ -167,7 +167,7 @@ public class NodePrioritizer { NodeType.tenant); PrioritizableNode nodePri = toPrioritizable(newNode, false, true); if ( ! nodePri.violatesSpares || isAllocatingForReplacement) { - log.log(LogLevel.DEBUG, "Adding new Docker node " + newNode); + log.log(Level.FINE, "Adding new Docker node " + newNode); nodes.put(newNode, nodePri); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index a74671d4e26..d2e214d9199 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -183,7 +183,7 @@ public class NodeRepositoryProvisioner implements Provisioner { nodes.sort(Comparator.comparingInt(node -> node.allocation().get().membership().index())); List hosts = new ArrayList<>(nodes.size()); for (Node node : nodes) { - log.log(LogLevel.DEBUG, () -> "Prepared node " + node.hostname() + " - " + node.flavor()); + log.log(Level.FINE, () -> "Prepared node " + node.hostname() + " - " + node.flavor()); Allocation nodeAllocation = node.allocation().orElseThrow(IllegalStateException::new); hosts.add(new HostSpec(node.hostname(), List.of(), @@ -194,7 +194,7 @@ public class NodeRepositoryProvisioner implements Provisioner { requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources), node.status().dockerImage())); if (nodeAllocation.networkPorts().isPresent()) { - log.log(LogLevel.DEBUG, () -> "Prepared node " + node.hostname() + " has port allocations"); + log.log(Level.FINE, () -> "Prepared node " + node.hostname() + " has port allocations"); } } return hosts; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java index d79accc5b36..72033e41192 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java @@ -62,7 +62,7 @@ public class StorageNodeImpl implements StorageNode { ConfigId configId = storageService.configId(); int nodeIndex = VespaModelUtil.getStorageNodeIndex(configId); - logger.log(LogLevel.DEBUG, () -> "Setting cluster controller state for " + + logger.log(Level.FINE, () -> "Setting cluster controller state for " + "application " + applicationInstance.reference().asString() + ", host " + hostName() + ", cluster name " + clusterId + diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index 5d4090fea30..9ed9e1b9063 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -78,10 +78,10 @@ public class HostResource implements HostApi { applicationUri.toString(), hostServices); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.DEBUG, "Failed to get host " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to get host " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("getHost", hostName, e); } catch (HostNameNotFoundException e) { - log.log(LogLevel.DEBUG, "Host not found: " + hostName); + log.log(Level.FINE, "Host not found: " + hostName); throw new NotFoundException(e); } } @@ -101,14 +101,14 @@ public class HostResource implements HostApi { try { orchestrator.setNodeStatus(hostName, state); } catch (HostNameNotFoundException e) { - log.log(LogLevel.DEBUG, "Host not found: " + hostName); + log.log(Level.FINE, "Host not found: " + hostName); throw new NotFoundException(e); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.DEBUG, "Failed to patch " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to patch " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("patch", hostName, e); } catch (OrchestrationException e) { String message = "Failed to set " + hostName + " to " + state + ": " + e.getMessage(); - log.log(LogLevel.DEBUG, message, e); + log.log(Level.FINE, message, e); throw new InternalServerErrorException(message); } } @@ -124,13 +124,13 @@ public class HostResource implements HostApi { try { orchestrator.suspend(hostName); } catch (HostNameNotFoundException e) { - log.log(LogLevel.DEBUG, "Host not found: " + hostName); + log.log(Level.FINE, "Host not found: " + hostName); throw new NotFoundException(e); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.DEBUG, "Failed to suspend " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to suspend " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("suspend", hostName, e); } catch (HostStateChangeDeniedException e) { - log.log(LogLevel.DEBUG, "Failed to suspend " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to suspend " + hostName + ": " + e.getMessage()); throw webExceptionWithDenialReason("suspend", hostName, e); } return new UpdateHostResponse(hostName.s(), null); @@ -142,13 +142,13 @@ public class HostResource implements HostApi { try { orchestrator.resume(hostName); } catch (HostNameNotFoundException e) { - log.log(LogLevel.DEBUG, "Host not found: " + hostName); + log.log(Level.FINE, "Host not found: " + hostName); throw new NotFoundException(e); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.DEBUG, "Failed to resume " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to resume " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("resume", hostName, e); } catch (HostStateChangeDeniedException e) { - log.log(LogLevel.DEBUG, "Failed to resume " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to resume " + hostName + ": " + e.getMessage()); throw webExceptionWithDenialReason("resume", hostName, e); } return new UpdateHostResponse(hostName.s(), null); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java index 11afb8a849c..3d60d40dfcf 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java @@ -43,21 +43,21 @@ public class HostSuspensionResource implements HostSuspensionApi { try { orchestrator.suspendAll(parentHostname, hostnames); } catch (BatchHostStateChangeDeniedException e) { - log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); + log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); + log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT); } catch (BatchHostNameNotFoundException e) { - log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); + log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); // Note that we're returning BAD_REQUEST instead of NOT_FOUND because the resource identified // by the URL path was found. It's one of the hostnames in the request it failed to find. throw createWebApplicationException(e.getMessage(), Response.Status.BAD_REQUEST); } catch (BatchInternalErrorException e) { - log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); + log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); throw createWebApplicationException(e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR); } - log.log(LogLevel.DEBUG, "Suspended " + hostnames + " with parent " + parentHostname); + log.log(Level.FINE, "Suspended " + hostnames + " with parent " + parentHostname); return BatchOperationResult.successResult(); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java index d50eb1c7e26..cbb6902066e 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java @@ -109,7 +109,7 @@ public class HostInfosServiceImpl implements HostInfosService { curator.framework().delete().forPath(path); return true; } catch (NoNodeException e) { - log.log(LogLevel.DEBUG, debugLogMessageIfNotExists, e); + log.log(Level.FINE, debugLogMessageIfNotExists, e); return false; } catch (Exception e) { throw new RuntimeException(e); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java index 27eea7a685b..21c3c846385 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java @@ -93,7 +93,7 @@ public class ZkStatusService implements StatusService { return resultSet; } catch (Exception e) { - log.log(LogLevel.DEBUG, "Something went wrong while listing out applications in suspend.", e); + log.log(Level.FINE, "Something went wrong while listing out applications in suspend.", e); throw new RuntimeException(e); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index 84aa7f5e72f..dea80254b05 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -42,7 +42,7 @@ public class DuperModel { void setComplete() { if (!isComplete) { - logger.log(LogLevel.DEBUG, "All applications have been activated: duper model is complete"); + logger.log(Level.FINE, "All applications have been activated: duper model is complete"); isComplete = true; listeners.forEach(DuperModelListener::bootstrapComplete); @@ -95,7 +95,7 @@ public class DuperModel { } else { logPrefix = isComplete ? "Reactivated application " : "Rebootstrapped application "; } - logger.log(LogLevel.DEBUG, logPrefix + id.toFullString()); + logger.log(Level.FINE, logPrefix + id.toFullString()); updateHostnameVsIdMaps(applicationInfo, id); @@ -110,7 +110,7 @@ public class DuperModel { ApplicationInfo application = applicationsById.remove(applicationId); if (application != null) { - logger.log(LogLevel.DEBUG, "Removed application " + applicationId.toFullString()); + logger.log(Level.FINE, "Removed application " + applicationId.toFullString()); listeners.forEach(listener -> listener.applicationRemoved(applicationId)); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index 552cb09da14..3c6ff8bd903 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -149,7 +149,7 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { case "storagenode": return Optional.of("storage/cluster." + configId.s()); default: - logger.log(LogLevel.DEBUG, "Unknown service type " + serviceType.s() + + logger.log(Level.FINE, "Unknown service type " + serviceType.s() + " with config id " + configId.s()); return Optional.empty(); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index cdd691b1ac7..6bb74e98fbd 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -170,7 +170,7 @@ class ClientFeederV3 { operationId = streamReaderV3.getNextOperationId(requestInputStream); } catch (IOException ioe) { if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, Exceptions.toMessageString(ioe), ioe); + log.log(Level.FINE, Exceptions.toMessageString(ioe), ioe); } return Optional.empty(); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java index 42cff1ff4e7..d5596d02804 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java @@ -153,7 +153,7 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib defaultAbortOnSendError = feederOptions.abortOnSendError(); if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received new config (" + + log.log(Level.FINE, "Received new config (" + "route: " + (defaultRoute != null ? defaultRoute : "") + ", timeout: " + defaultTimeoutMillis + " ms, retry enabled: " + retryEnabled + ", docproc chain: " + (defaultDocprocChain != null ? defaultDocprocChain : "") + diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java index 9b83e1d09cb..54307f01258 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java @@ -128,14 +128,14 @@ public class SharedSender implements ReplyHandler { log.log(LogLevel.SPAM, "Received reply for file " + owner.toString() + " count was " + owner.getPending().val()); } if (owner.isAborted()) { - log.log(LogLevel.DEBUG, "Received reply for file " + owner.toString() + " which is aborted"); + log.log(Level.FINE, "Received reply for file " + owner.toString() + " which is aborted"); owner.getPending().clear(); return; } if (owner.handleReply(r)) { owner.getPending().dec(); } else { - log.log(LogLevel.DEBUG, "Received reply for file " + owner.toString() + " which wants to abort"); + log.log(Level.FINE, "Received reply for file " + owner.toString() + " which wants to abort"); owner.getPending().clear(); } } else { diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java index 509a588dac9..a1dec70f9df 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java @@ -208,7 +208,7 @@ public class VdsVisitTarget { @SuppressWarnings("unchecked") public void run() throws Exception { initShutdownHook(); - log.log(LogLevel.DEBUG, "Starting VdsVisitTarget"); + log.log(Level.FINE, "Starting VdsVisitTarget"); MessageBusParams mbusParams = new MessageBusParams(new LoadTypeSet()); mbusParams.getRPCNetworkParams().setIdentity(new Identity(slobrokAddress)); diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java index df1ab24901f..d71e23a0623 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java @@ -39,9 +39,9 @@ class CuratorCompletionWaiter implements Curator.CompletionWaiter { public void awaitCompletion(Duration timeout) { List respondents; try { - log.log(LogLevel.DEBUG, "Synchronizing on barrier " + barrierPath); + log.log(Level.FINE, "Synchronizing on barrier " + barrierPath); respondents = awaitInternal(timeout); - log.log(LogLevel.DEBUG, "Done synchronizing on barrier " + barrierPath); + log.log(Level.FINE, "Done synchronizing on barrier " + barrierPath); } catch (Exception e) { throw new RuntimeException(e); } -- cgit v1.2.3 From 7786b391540a5632a17536e4b43ef635553db21a Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 15:33:24 +0200 Subject: LogLevel.SPAM -> Level.FINEST --- .../yahoo/vespa/clustercontroller/core/FleetController.java | 2 +- .../vespa/clustercontroller/core/MasterElectionHandler.java | 2 +- .../java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java | 8 ++++---- .../yahoo/vespa/clustercontroller/core/StateChangeHandler.java | 2 +- .../clustercontroller/core/database/MasterDataGatherer.java | 2 +- .../clustercontroller/core/database/ZooKeeperDatabase.java | 2 +- .../yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java | 6 +++--- .../core/status/statuspage/StatusPageServer.java | 2 +- .../com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java | 2 +- .../yahoo/config/model/producer/AbstractConfigProducer.java | 4 ++-- .../java/com/yahoo/vespa/model/builder/UserConfigBuilder.java | 2 +- .../com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java | 10 +++++----- .../com/yahoo/vespa/config/proxy/DelayedResponseHandler.java | 4 ++-- .../com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java | 6 +++--- .../main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java | 2 +- .../vespa/config/server/deploy/DeployHandlerLoggerTest.java | 2 +- .../java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java | 8 ++++---- docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java | 2 +- .../src/main/java/com/yahoo/documentapi/ProgressToken.java | 4 ++-- .../src/main/java/com/yahoo/documentapi/VisitorIterator.java | 4 ++-- .../java/com/yahoo/vespa/filedistribution/FileReceiver.java | 2 +- messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java | 2 +- .../src/main/java/com/yahoo/feedapi/SharedSender.java | 2 +- 23 files changed, 41 insertions(+), 41 deletions(-) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 1e66c397b88..4ae0fef015b 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -416,7 +416,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd */ public void handleFleetData(Map data) { verifyInControllerThread(); - log.log(LogLevel.SPAM, "Sending fleet data event on to master election handler"); + log.log(Level.FINEST, "Sending fleet data event on to master election handler"); metricUpdater.updateMasterElectionMetrics(data); masterElectionHandler.handleFleetData(data); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java index 8299e77e01a..1bfd86e9bc9 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java @@ -109,7 +109,7 @@ public class MasterElectionHandler implements MasterInterface { if (totalCount == 1) return false; // No point in doing master election with only one node configured to be cluster controller if (nextMasterData == null) { if (masterCandidate == null) { - log.log(LogLevel.SPAM, "Cluster controller " + index + ": No current master candidate. Waiting for data to do master election."); + log.log(Level.FINEST, "Cluster controller " + index + ": No current master candidate. Waiting for data to do master election."); } return false; // Nothing have happened since last time. } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java index 12cfe979511..12a6d72d400 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java @@ -342,19 +342,19 @@ abstract public class NodeInfo implements Comparable { } else { nextAttemptTime = time + 5000; } - log.log(LogLevel.SPAM, "Failed to get state from node " + toString() + ", scheduling next attempt in " + (nextAttemptTime - time) + " ms."); + log.log(Level.FINEST, "Failed to get state from node " + toString() + ", scheduling next attempt in " + (nextAttemptTime - time) + " ms."); } else { connectionAttemptCount = 0; timeOfFirstFailingConnectionAttempt = 0; reportedState = state; if (version == 0 || state.getState().equals(State.STOPPING)) { nextAttemptTime = time + cluster.getPollingFrequency(); - log.log(LogLevel.SPAM, "Scheduling next attempt to get state from " + toString() + " in " + (nextAttemptTime - time) + " ms (polling freq)."); + log.log(Level.FINEST, "Scheduling next attempt to get state from " + toString() + " in " + (nextAttemptTime - time) + " ms (polling freq)."); } else { nextAttemptTime = time; } } - log.log(LogLevel.SPAM, "Set reported state of node " + this + " to " + reportedState + ". Next connection attempt is at " + nextAttemptTime); + log.log(Level.FINEST, "Set reported state of node " + this + " to " + reportedState + ". Next connection attempt is at " + nextAttemptTime); } /** Sets the wanted state. The wanted state is taken as UP if a null argument is given */ @@ -374,7 +374,7 @@ abstract public class NodeInfo implements Comparable { } } wantedState = newWanted; - log.log(LogLevel.SPAM, "Set wanted state of node " + this + " to " + wantedState + "."); + log.log(Level.FINEST, "Set wanted state of node " + this + " to " + wantedState + "."); } public long getTimeForNextStateRequestAttempt() { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index e9005509c52..85895b49efd 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -426,7 +426,7 @@ public class StateChangeHandler { if (initializationProgressHasIncreased(currentState, reportedState)) { node.setInitProgressTime(timeNow); if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Reset initialize timer on " + node + " to " + node.getInitProgressTime()); + log.log(Level.FINEST, "Reset initialize timer on " + node + " to " + node.getInitProgressTime()); } } if (handleImplicitCrashEdgeFromReverseInitProgress(node, currentState, reportedState, nodeListener, timeNow)) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java index 28527162b71..8daa132ee5a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java @@ -138,7 +138,7 @@ public class MasterDataGatherer { } for(Integer vote : nextMasterData.values()) { if (vote == null) { - log.log(LogLevel.SPAM, "Fleetcontroller " + nodeIndex + ": Still not received votes from all fleet controllers. Awaiting more responses."); + log.log(Level.FINEST, "Fleetcontroller " + nodeIndex + ": Still not received votes from all fleet controllers. Awaiting more responses."); return; } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java index a2d370a6fe8..834ecab9f5b 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java @@ -103,7 +103,7 @@ public class ZooKeeperDatabase extends Database { try{ this.listener = zksl; setupRoot(); - log.log(LogLevel.SPAM, "Fleetcontroller " + nodeIndex + ": Asking for initial data on master election"); + log.log(Level.FINEST, "Fleetcontroller " + nodeIndex + ": Asking for initial data on master election"); masterDataGatherer = new MasterDataGatherer(session, zooKeeperRoot, listener, nodeIndex); completedOk = true; } finally { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index 5af1af519bb..643a6a94c50 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -74,7 +74,7 @@ public class SlobrokClient implements NodeLookup { freshMirror = false; } else if (cluster.getSlobrokGenerationCount() == mirrorVersion) { if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Slobrok still at generation count " + cluster.getSlobrokGenerationCount() + ". Not updating."); + log.log(Level.FINEST, "Slobrok still at generation count " + cluster.getSlobrokGenerationCount() + ". Not updating."); } return false; } @@ -151,7 +151,7 @@ public class SlobrokClient implements NodeLookup { nodeInfo.markRpcAddressLive(); } } - log.log(LogLevel.SPAM, "Slobrok information updated to generation " + cluster.getSlobrokGenerationCount()); + log.log(Level.FINEST, "Slobrok information updated to generation " + cluster.getSlobrokGenerationCount()); return true; } @@ -196,7 +196,7 @@ public class SlobrokClient implements NodeLookup { private Map getSlobrokData(String pattern) { Map result = new TreeMap<>(); List entries = mirror.lookup(pattern); - log.log(LogLevel.SPAM, "Looking for slobrok entries with pattern '" + pattern + "'. Found " + entries.size() + " entries."); + log.log(Level.FINEST, "Looking for slobrok entries with pattern '" + pattern + "'. Found " + entries.size() + " entries."); for (Mirror.Entry entry : entries) { StringTokenizer st = new StringTokenizer(entry.getName(), "/"); String addressType = st.nextToken(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index 8e6cbc28a85..19181f2d1eb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -58,7 +58,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { if (ssocket != null && ssocket.isBound() && (ssocket.getLocalPort() == port || port == 0)) { return true; } else { - log.log(LogLevel.SPAM, "Status page server socket is no longer connected: "+ (ssocket != null) + " " + ssocket.isBound() + " " + ssocket.getLocalPort() + " " + port); + log.log(Level.FINEST, "Status page server socket is no longer connected: "+ (ssocket != null) + " " + ssocket.isBound() + " " + ssocket.getLocalPort() + " " + port); return false; } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java index d6959ed1952..a596df164af 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java @@ -402,7 +402,7 @@ public class DummyVdsNode { private void rpc_storageConnect(Request req) { synchronized(timer) { - log.log(LogLevel.SPAM, "Dummy node " + this + " got old type handle connect message."); + log.log(Level.FINEST, "Dummy node " + this + " got old type handle connect message."); req.returnValues().add(new Int32Value(0)); negotiatedHandle = true; } diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index fc602ca8dcd..4f811f90ed9 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -218,13 +218,13 @@ public abstract class AbstractConfigProducer builderMap = new LinkedHashMap<>(); if (producerSpec == null) { - log.log(LogLevel.SPAM, "In getUserConfigs. producerSpec is null"); + log.log(Level.FINEST, "In getUserConfigs. producerSpec is null"); } log.log(Level.FINE, "getUserConfigs for " + producerSpec); for (Element configE : XML.getChildren(producerSpec, "config")) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index c8b16f5376a..1399c3677af 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -247,10 +247,10 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer private void dispatchRpcRequest(Request request, Runnable handler) { request.detach(); - log.log(LogLevel.SPAM, () -> String.format("Dispatching RPC request %s", requestLogId(request))); + log.log(Level.FINEST, () -> String.format("Dispatching RPC request %s", requestLogId(request))); rpcExecutor.execute(() -> { try { - log.log(LogLevel.SPAM, () -> String.format("Executing RPC request %s.", requestLogId(request))); + log.log(Level.FINEST, () -> String.format("Executing RPC request %s.", requestLogId(request))); handler.run(); } catch (Exception e) { log.log(LogLevel.WARNING, @@ -280,11 +280,11 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer try { RawConfig config = proxyServer.resolveConfig(request); if (config == null) { - log.log(LogLevel.SPAM, () -> "No config received yet for " + request.getShortDescription() + ", not sending response"); + log.log(Level.FINEST, () -> "No config received yet for " + request.getShortDescription() + ", not sending response"); } else if (ProxyServer.configOrGenerationHasChanged(config, request)) { returnOkResponse(request, config); } else { - log.log(LogLevel.SPAM, "No new config for " + request.getShortDescription() + ", not sending response"); + log.log(Level.FINEST, "No new config for " + request.getShortDescription() + ", not sending response"); } } catch (Exception e) { e.printStackTrace(); @@ -353,7 +353,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer request.addOkResponse(config.getPayload(), config.getGeneration(), config.isInternalRedeploy(), config.getConfigMd5()); log.log(Level.FINE, () -> "Return response: " + request.getShortDescription() + ",configMd5=" + config.getConfigMd5() + ",generation=" + config.getGeneration()); - log.log(LogLevel.SPAM, () -> "Config payload in response for " + request.getShortDescription() + ":" + config.getPayload()); + log.log(Level.FINEST, () -> "Config payload in response for " + request.getShortDescription() + ":" + config.getPayload()); // TODO Catch exception for now, since the request might have been returned in CheckDelayedResponse diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java index d9d2de2fac2..42bb2d21b55 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java @@ -38,7 +38,7 @@ public class DelayedResponseHandler implements Runnable { void checkDelayedResponses() { try { long start = System.currentTimeMillis(); - log.log(LogLevel.SPAM, () -> "Running DelayedResponseHandler. There are " + delayedResponses.size() + + log.log(Level.FINEST, () -> "Running DelayedResponseHandler. There are " + delayedResponses.size() + " delayed responses. First one is " + delayedResponses.responses().peek()); DelayedResponse response; AtomicInteger i = new AtomicInteger(0); @@ -54,7 +54,7 @@ public class DelayedResponseHandler implements Runnable { request.getConfigKey() + ", will retry"); } } - log.log(LogLevel.SPAM, () -> "Finished running DelayedResponseHandler. " + i.get() + " delayed responses sent in " + + log.log(Level.FINEST, () -> "Finished running DelayedResponseHandler. " + i.get() + " delayed responses sent in " + (System.currentTimeMillis() - start) + " ms"); } catch (Exception e) { // To avoid thread throwing exception and executor never running this again log.log(LogLevel.WARNING, "Got exception in DelayedResponseHandler: " + Exceptions.toMessageString(e)); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index d37408f97b3..31fd8a2682a 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -134,9 +134,9 @@ class RpcConfigSourceClient implements ConfigSourceClient { if (cachedConfig != null) { log.log(Level.FINE, () -> "Found config " + configCacheKey + " in cache, generation=" + cachedConfig.getGeneration() + ",configmd5=" + cachedConfig.getConfigMd5()); - log.log(LogLevel.SPAM, () -> "input config=" + input + ",cached config=" + cachedConfig); + log.log(Level.FINEST, () -> "input config=" + input + ",cached config=" + cachedConfig); if (ProxyServer.configOrGenerationHasChanged(cachedConfig, request)) { - log.log(LogLevel.SPAM, () -> "Cached config is not equal to requested, will return it"); + log.log(Level.FINEST, () -> "Cached config is not equal to requested, will return it"); if (delayedResponses.remove(delayedResponse)) { // unless another thread already did it ret = cachedConfig; @@ -217,7 +217,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { public void updateSubscribers(RawConfig config) { log.log(Level.FINE, () -> "Config updated for " + config.getKey() + "," + config.getGeneration()); DelayQueue responseDelayQueue = delayedResponses.responses(); - log.log(LogLevel.SPAM, () -> "Delayed response queue: " + responseDelayQueue); + log.log(Level.FINEST, () -> "Delayed response queue: " + responseDelayQueue); if (responseDelayQueue.size() == 0) { log.log(Level.FINE, () -> "There exists no matching element on delayed response queue for " + config.getKey()); return; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 6082e4d34e5..e5918d19bc0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -157,7 +157,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { */ private void getConfigV3(Request req) { if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, getConfigMethodName); + log.log(Level.FINEST, getConfigMethodName); } req.detach(); rpcAuthorizer.authorizeConfigRequest(req) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java index 9e7576ae8f5..a52788031c7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java @@ -43,7 +43,7 @@ public class DeployHandlerLoggerTest { private void logMessages(DeployLogger logger) { logger.log(Level.FINE, "foobar"); - logger.log(LogLevel.SPAM, "foobar"); + logger.log(Level.FINEST, "foobar"); logger.log(LogLevel.FINE, "baz"); logger.log(LogLevel.WARNING, "baz"); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index f7c8c646bb4..edfc12f3efa 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -385,7 +385,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { public void onSearchResult(SearchResult sr) { if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Got SearchResult for query with selection " + params.getDocumentSelection()); + log.log(Level.FINEST, "Got SearchResult for query with selection " + params.getDocumentSelection()); } handleSearchResult(sr); } @@ -413,14 +413,14 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private void mergeGroupingMaps(Map newGroupingMap) { if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "mergeGroupingMaps: newGroupingMap = " + newGroupingMap); + log.log(Level.FINEST, "mergeGroupingMaps: newGroupingMap = " + newGroupingMap); } for(Integer key : newGroupingMap.keySet()) { byte [] value = newGroupingMap.get(key); Grouping newGrouping = new Grouping(); if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Received group with key " + key + " and size " + value.length); + log.log(Level.FINEST, "Received group with key " + key + " and size " + value.length); } BufferSerializer buf = new BufferSerializer( new GrowableByteBuffer(ByteBuffer.wrap(value)) ); newGrouping.deserialize(buf); @@ -441,7 +441,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { public void onDocumentSummary(DocumentSummary ds) { if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Got DocumentSummary for query with selection " + params.getDocumentSelection()); + log.log(Level.FINEST, "Got DocumentSummary for query with selection " + params.getDocumentSelection()); } handleSummary(ds); } diff --git a/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java b/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java index fd1b81fce40..ae636e12ba9 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java @@ -150,7 +150,7 @@ public class DocprocExecutor { } } message.append("]"); - log.log(LogLevel.SPAM, message.toString()); + log.log(Level.FINEST, message.toString()); } /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java index eea3d6e66e8..b7c78e28dbe 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java @@ -368,7 +368,7 @@ public class ProgressToken { return; } if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Adding bucket pair " + superbucket + log.log(Level.FINEST, "Adding bucket pair " + superbucket + ":" + progress + " with state " + state); } @@ -790,7 +790,7 @@ public class ProgressToken { if (rightSibling != null) { assert(rightSibling.getState() == BucketState.BUCKET_PENDING); if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Merging " + bucket + " with rhs " + rightCheck); + log.log(Level.FINEST, "Merging " + bucket + " with rhs " + rightCheck); } // If right sibling has progress, it will unfortunately have to // be discarded diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java index ecf6d640ef7..05324d957b8 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java @@ -246,7 +246,7 @@ public class VisitorIterator { lastMergedBucket.getId() | (1L << (lastMergedBucket.getUsedBits() - 1))); if (pending.equals(rightCheck)) { if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Skipped " + pending + + log.log(Level.FINEST, "Skipped " + pending + ", as it was right sibling of " + lastMergedBucket); } continue; @@ -286,7 +286,7 @@ public class VisitorIterator { } } if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "New range bucket cursor is " + + log.log(Level.FINEST, "New range bucket cursor is " + progressToken.getBucketCursor()); } } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index a79f3831c17..2db66f6c859 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -255,7 +255,7 @@ public class FileReceiver { retval = 1; } double completeness = (double) session.currentFileSize / (double) session.fileSize; - log.log(LogLevel.SPAM, () -> String.format("%.1f percent of '%s' downloaded", completeness * 100, reference.value())); + log.log(Level.FINEST, () -> String.format("%.1f percent of '%s' downloaded", completeness * 100, reference.value())); downloader.setDownloadStatus(reference, completeness); req.returnValues().add(new Int32Value(retval)); } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java b/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java index 77bbd662cc4..d90cd2fb3da 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java @@ -61,7 +61,7 @@ public class SendProxy implements MessageHandler, ReplyHandler { if (reply.hasErrors()) { log.log(Level.FINE, "Trace for reply with error(s):\n" + reply.getTrace()); } else if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Trace for reply:\n" + reply.getTrace()); + log.log(Level.FINEST, "Trace for reply:\n" + reply.getTrace()); } Trace empty = new Trace(); trace.swap(empty); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java index 54307f01258..aa2849a027b 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java @@ -125,7 +125,7 @@ public class SharedSender implements ReplyHandler { metrics.addReply(r); } if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Received reply for file " + owner.toString() + " count was " + owner.getPending().val()); + log.log(Level.FINEST, "Received reply for file " + owner.toString() + " count was " + owner.getPending().val()); } if (owner.isAborted()) { log.log(Level.FINE, "Received reply for file " + owner.toString() + " which is aborted"); -- cgit v1.2.3 From 816e0e5dcd0987e8be4e3ea66402c83b770dd467 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 15:37:33 +0200 Subject: LogLevel.INFO -> Level.INFO --- .../ConfigserverSslContextFactoryProvider.java | 6 +-- .../ca/restapi/CertificateAuthorityApiHandler.java | 2 +- .../clustercontroller/core/FleetController.java | 8 ++-- .../core/MasterElectionHandler.java | 26 +++++------ .../core/SystemStateBroadcaster.java | 2 +- .../core/database/DatabaseHandler.java | 10 ++--- .../core/database/MasterDataGatherer.java | 12 ++--- .../core/database/ZooKeeperDatabase.java | 16 +++---- .../clustercontroller/core/rpc/RpcServer.java | 4 +- .../core/status/statuspage/StatusPageServer.java | 6 +-- .../core/FleetControllerTest.java | 2 +- .../clustercontroller/core/MasterElectionTest.java | 52 +++++++++++----------- .../clustercontroller/core/RpcServerTest.java | 2 +- .../vespa/clustercontroller/core/SlobrokTest.java | 14 +++--- .../clustercontroller/core/testutils/Waiter.java | 6 +-- .../provider/FilesApplicationPackage.java | 4 +- .../java/com/yahoo/vespa/model/VespaModel.java | 2 +- .../model/builder/xml/dom/DomAdminV2Builder.java | 2 +- .../model/builder/xml/dom/DomAdminV4Builder.java | 2 +- .../yahoo/vespa/model/content/StorageGroup.java | 2 +- .../com/yahoo/vespa/config/proxy/MemoryCache.java | 6 +-- .../com/yahoo/vespa/config/proxy/ProxyServer.java | 2 +- .../vespa/config/proxy/RpcConfigSourceClient.java | 8 ++-- .../filedistribution/CachedFilesMaintainer.java | 2 +- .../FileDistributionRpcServer.java | 2 +- .../filedistribution/UrlDownloadRpcServer.java | 6 +-- .../vespa/config/server/ApplicationRepository.java | 12 ++--- .../vespa/config/server/ConfigServerBootstrap.java | 10 ++--- .../vespa/config/server/SuperModelManager.java | 2 +- .../server/application/TenantApplications.java | 2 +- .../vespa/config/server/deploy/Deployment.java | 2 +- .../config/server/http/HttpErrorResponse.java | 2 +- .../config/server/modelfactory/ModelsBuilder.java | 4 +- .../yahoo/vespa/config/server/rpc/RpcServer.java | 2 +- .../rpc/security/MultiTenantRpcAuthorizer.java | 2 +- .../vespa/config/server/session/RemoteSession.java | 4 +- .../config/server/session/RemoteSessionRepo.java | 4 +- .../server/session/SessionZooKeeperClient.java | 4 +- .../vespa/config/server/tenant/TenantDebugger.java | 2 +- .../config/server/tenant/TenantRepository.java | 6 +-- .../config/server/tenant/TenantRequestHandler.java | 2 +- .../yahoo/container/jdisc/state/StateMonitor.java | 2 +- .../container/jdisc/ConfiguredApplication.java | 4 +- .../jdisc/messagebus/MbusServerProvider.java | 2 +- .../certificate/EndpointCertificateManager.java | 8 ++-- .../controller/dns/NameServiceForwarder.java | 2 +- .../maintenance/NameServiceDispatcher.java | 2 +- .../restapi/filter/AthenzRoleFilter.java | 2 +- .../yahoo/vespa/hosted/dockerapi/DockerImpl.java | 4 +- .../messagebus/MessageBusAsyncSession.java | 2 +- .../fileacquirer/FileAcquirerImpl.java | 2 +- .../yahoo/vespa/filedistribution/FileReceiver.java | 2 +- .../yahoo/vespa/filedistribution/RpcTester.java | 16 +++---- .../com/yahoo/vespa/flags/file/FlagDbFile.java | 6 +-- .../com/yahoo/logserver/AbstractPluginLoader.java | 2 +- .../main/java/com/yahoo/messagebus/MessageBus.java | 2 +- .../yahoo/messagebus/network/rpc/RPCTarget.java | 2 +- .../metricsproxy/service/HttpMetricFetcher.java | 4 +- .../maintenance/ApplicationMaintainer.java | 2 +- .../maintenance/InfrastructureProvisioner.java | 2 +- .../maintenance/MaintenanceDeployment.java | 2 +- .../hosted/provision/maintenance/NodeFailer.java | 2 +- .../persistence/CuratorDatabaseClient.java | 6 +-- .../provision/provisioning/InfraDeployerImpl.java | 2 +- .../resources/ApplicationSuspensionResource.java | 12 ++--- .../vespa/service/duper/DuperModelManager.java | 4 +- .../com/yahoo/vespavisit/StdOutVisitorHandler.java | 2 +- .../vespa/zookeeper/VespaZooKeeperServerImpl.java | 2 +- 68 files changed, 183 insertions(+), 183 deletions(-) diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index 54e7a4d2947..a4d9ded9254 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -148,7 +148,7 @@ public class ConfigserverSslContextFactoryProvider extends TlsContextBasedProvid SiaUtils.writePrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity, privateKey); Instant expirationTime = certificate.getNotAfter().toInstant(); Duration expiry = Duration.between(certificate.getNotBefore().toInstant(), expirationTime); - log.log(LogLevel.INFO, String.format("Got Athenz x509 certificate with expiry %s (expires %s)", expiry, expirationTime)); + log.log(Level.INFO, String.format("Got Athenz x509 certificate with expiry %s (expires %s)", expiry, expirationTime)); return KeyStoreBuilder.withType(KeyStoreType.JKS) .withKeyEntry(CERTIFICATE_ALIAS, privateKey, keystorePwd, certificate) .build(); @@ -168,11 +168,11 @@ public class ConfigserverSslContextFactoryProvider extends TlsContextBasedProvid @Override public void run() { try { - log.log(LogLevel.INFO, "Updating configserver provider certificate from ZTS"); + log.log(Level.INFO, "Updating configserver provider certificate from ZTS"); char[] keystorePwd = generateKeystorePassword(); KeyStore keyStore = updateKeystore(configserverIdentity, keystorePwd, keyProvider, ztsClient, athenzProviderServiceConfig); keyManager.updateKeystore(keyStore, keystorePwd); - log.log(LogLevel.INFO, "Certificate successfully updated"); + log.log(Level.INFO, "Certificate successfully updated"); } catch (Throwable t) { log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + t.getMessage(), t); } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index dfbad290fbb..0325971038f 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -101,7 +101,7 @@ public class CertificateAuthorityApiHandler extends LoggingRequestHandler { confirmation.set(InstanceValidator.SAN_IPS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRegistration.csr(), SubjectAlternativeName.Type.IP_ADDRESS)); confirmation.set(InstanceValidator.SAN_DNS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRegistration.csr(), SubjectAlternativeName.Type.DNS_NAME)); if (!instanceValidator.isValidInstance(confirmation)) { - log.log(LogLevel.INFO, "Invalid instance registration for " + instanceRegistration.toString()); + log.log(Level.INFO, "Invalid instance registration for " + instanceRegistration.toString()); return ErrorResponse.forbidden("Unable to launch service: " +instanceRegistration.service()); } var certificate = certificates.create(instanceRegistration.csr(), caCertificate(), caPrivateKey()); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 4ae0fef015b..bfa202432b6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -299,12 +299,12 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd public void shutdown() throws InterruptedException, java.io.IOException { if (runner != null && isRunning()) { - log.log(LogLevel.INFO, "Joining event thread."); + log.log(Level.INFO, "Joining event thread."); running.set(false); synchronized(monitor) { monitor.notifyAll(); } runner.join(); } - log.log(LogLevel.INFO, "Fleetcontroller done shutting down event thread."); + log.log(Level.INFO, "Fleetcontroller done shutting down event thread."); controllerThreadId = Thread.currentThread().getId(); database.shutdown(this); @@ -321,7 +321,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd public void updateOptions(FleetControllerOptions options, long configGeneration) { synchronized(monitor) { assert(this.options.fleetControllerIndex == options.fleetControllerIndex); - log.log(LogLevel.INFO, "Fleetcontroller " + options.fleetControllerIndex + " has new options"); + log.log(Level.INFO, "Fleetcontroller " + options.fleetControllerIndex + " has new options"); nextOptions = options.clone(); nextConfigGeneration = configGeneration; monitor.notifyAll(); @@ -1125,7 +1125,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } } if (ackedNodes >= nodeCount) { - log.log(LogLevel.INFO, ackedNodes + " nodes now have acked system state " + version + " or higher."); + log.log(Level.INFO, ackedNodes + " nodes now have acked system state " + version + " or higher."); return; } long remainingTime = maxTime - System.currentTimeMillis(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java index 1bfd86e9bc9..7b026dd6a74 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java @@ -116,25 +116,25 @@ public class MasterElectionHandler implements MasterInterface { // Move next data to temporary, such that we don't need to keep lock, and such that we don't retry // if we happen to fail processing the data. Map state; - log.log(LogLevel.INFO, "Cluster controller " + index + ": Handling new master election, as we have received " + nextMasterData.size() + " entries"); + log.log(Level.INFO, "Cluster controller " + index + ": Handling new master election, as we have received " + nextMasterData.size() + " entries"); synchronized (monitor) { state = nextMasterData; nextMasterData = null; } - log.log(LogLevel.INFO, "Cluster controller " + index + ": Got master election state " + toString(state) + "."); + log.log(Level.INFO, "Cluster controller " + index + ": Got master election state " + toString(state) + "."); if (state.isEmpty()) throw new IllegalStateException("Database has no master data. We should at least have data for ourselves."); Map.Entry first = state.entrySet().iterator().next(); Integer currentMaster = getMaster(); if (currentMaster != null && first.getKey().intValue() != currentMaster.intValue()) { - log.log(LogLevel.INFO, "Cluster controller " + index + ": Master gone from ZooKeeper. Tagging timestamp. Will wait " + this.masterZooKeeperCooldownPeriod + " ms."); + log.log(Level.INFO, "Cluster controller " + index + ": Master gone from ZooKeeper. Tagging timestamp. Will wait " + this.masterZooKeeperCooldownPeriod + " ms."); masterGoneFromZooKeeperTime = timer.getCurrentTimeInMillis(); masterCandidate = null; } if (first.getValue().intValue() != first.getKey().intValue()) { - log.log(LogLevel.INFO, "Fleet controller " + index + ": First index is not currently trying to become master. Waiting for it to change state"); + log.log(Level.INFO, "Fleet controller " + index + ": First index is not currently trying to become master. Waiting for it to change state"); masterCandidate = null; if (first.getKey() == index) { - log.log(LogLevel.INFO, "Cluster controller " + index + ": We are next in line to become master. Altering our state to look for followers"); + log.log(Level.INFO, "Cluster controller " + index + ": We are next in line to become master. Altering our state to look for followers"); database.setMasterVote(dbContext, index); } } else { @@ -148,21 +148,21 @@ public class MasterElectionHandler implements MasterInterface { if (2 * followers > totalCount) { Integer newMaster = getMaster(); if (newMaster != null && currentMaster != null && newMaster.intValue() == currentMaster.intValue()) { - log.log(LogLevel.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + currentMaster + " is still the master"); + log.log(Level.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + currentMaster + " is still the master"); } else if (newMaster != null && currentMaster != null) { - log.log(LogLevel.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + newMaster + " took over for fleet controller " + currentMaster + " as master"); + log.log(Level.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + newMaster + " took over for fleet controller " + currentMaster + " as master"); } else if (newMaster == null) { - log.log(LogLevel.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + masterCandidate + " is new master candidate, but needs to wait before it can take over"); + log.log(Level.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + masterCandidate + " is new master candidate, but needs to wait before it can take over"); } else { - log.log(LogLevel.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + newMaster + " is newly elected master"); + log.log(Level.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + newMaster + " is newly elected master"); } } else { - log.log(LogLevel.INFO, "MASTER_ELECTION: Cluster controller " + index + ": Currently too few followers for cluster controller candidate " + masterCandidate + ". No current master. (" + followers + "/" + totalCount + " followers)"); + log.log(Level.INFO, "MASTER_ELECTION: Cluster controller " + index + ": Currently too few followers for cluster controller candidate " + masterCandidate + ". No current master. (" + followers + "/" + totalCount + " followers)"); } Integer ourState = state.get(index); if (ourState == null) throw new IllegalStateException("Database lacks data from ourselves. This should always be present."); if (ourState.intValue() != first.getKey().intValue()) { - log.log(LogLevel.INFO, "Cluster controller " + index + ": Altering our state to follow new fleet controller master candidate " + first.getKey()); + log.log(Level.INFO, "Cluster controller " + index + ": Altering our state to follow new fleet controller master candidate " + first.getKey()); database.setMasterVote(dbContext, first.getKey()); } } @@ -209,7 +209,7 @@ public class MasterElectionHandler implements MasterInterface { } public void handleFleetData(Map data) { - log.log(LogLevel.INFO, "Cluster controller " + index + ": Got new fleet data with " + data.size() + " entries: " + data); + log.log(Level.INFO, "Cluster controller " + index + ": Got new fleet data with " + data.size() + " entries: " + data); synchronized (monitor) { nextMasterData = data; monitor.notifyAll(); @@ -218,7 +218,7 @@ public class MasterElectionHandler implements MasterInterface { public void lostDatabaseConnection() { if (totalCount > 1) { - log.log(LogLevel.INFO, "Cluster controller " + index + ": Clearing master data as we lost connection on node " + index); + log.log(Level.INFO, "Cluster controller " + index + ": Clearing master data as we lost connection on node " + index); masterData = null; masterCandidate = null; followers = 0; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java index d6b8364954b..645ad694a0c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java @@ -263,7 +263,7 @@ public class SystemStateBroadcaster { ClusterState baselineState = clusterStateBundle.getBaselineClusterState(); if (!baselineState.isOfficial()) { - log.log(LogLevel.INFO, String.format("Publishing cluster state version %d", baselineState.getVersion())); + log.log(Level.INFO, String.format("Publishing cluster state version %d", baselineState.getVersion())); baselineState.setOfficial(true); // FIXME this violates state bundle immutability } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java index a7570ce5028..f648f08a3ea 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java @@ -115,7 +115,7 @@ public class DatabaseHandler { synchronized (databaseMonitor) { wasRunning = database != null; if (wasRunning) { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Resetting database state"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Resetting database state"); database.close(); database = null; } @@ -123,7 +123,7 @@ public class DatabaseHandler { clearSessionMetaData(); if (wasRunning) { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Done resetting database state"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Done resetting database state"); } } @@ -141,7 +141,7 @@ public class DatabaseHandler { if (address == null && zooKeeperAddress == null) return; if (address != null && zooKeeperAddress != null && address.equals(zooKeeperAddress)) return; if (zooKeeperAddress != null) { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": " + (address == null ? "Stopped using ZooKeeper." : "Got new ZooKeeper address to use: " + address)); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": " + (address == null ? "Stopped using ZooKeeper." : "Got new ZooKeeper address to use: " + address)); } zooKeeperAddress = address; reset(); @@ -165,7 +165,7 @@ public class DatabaseHandler { } // We still hold the database lock while calling this, we want to block callers. clearSessionMetaData(); - log.log(LogLevel.INFO, + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Setting up new ZooKeeper session at " + zooKeeperAddress); DatabaseFactory.Params params = new DatabaseFactory.Params() .cluster(cluster) @@ -189,7 +189,7 @@ public class DatabaseHandler { log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to connect to ZooKeeper at " + zooKeeperAddress + " with session timeout " + zooKeeperSessionTimeout + ": " + sw); } - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Done setting up new ZooKeeper session at " + zooKeeperAddress); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Done setting up new ZooKeeper session at " + zooKeeperAddress); } /** diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java index 8daa132ee5a..1c50afa4310 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java @@ -51,11 +51,11 @@ public class MasterDataGatherer { public void process(WatchedEvent watchedEvent) { switch (watchedEvent.getType()) { case NodeChildrenChanged: // Fleetcontrollers have either connected or disconnected to ZooKeeper - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": A change occurred in the list of registered fleetcontrollers. Requesting new information"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": A change occurred in the list of registered fleetcontrollers. Requesting new information"); session.getChildren(zooKeeperRoot + "indexes", this, childListener, null); break; case NodeDataChanged: // A fleetcontroller has changed what node it is voting for - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Altered data in node " + watchedEvent.getPath() + ". Requesting new vote"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Altered data in node " + watchedEvent.getPath() + ". Requesting new vote"); int index = getIndex(watchedEvent.getPath()); synchronized (nextMasterData) { nextMasterData.put(index, null); @@ -84,7 +84,7 @@ public class MasterDataGatherer { private class DirCallback implements AsyncCallback.ChildrenCallback { public void processResult(int version, String path, Object context, List nodes) { if (nodes == null) nodes = new LinkedList(); - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Got node list response from " + path + " version " + version + " with " + nodes.size() + " nodes"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Got node list response from " + path + " version " + version + " with " + nodes.size() + " nodes"); synchronized (nextMasterData) { nextMasterData.clear(); for (String node : nodes) { @@ -105,14 +105,14 @@ public class MasterDataGatherer { public void processResult(int code, String path, Object context, byte[] rawdata, Stat stat) { String data = rawdata == null ? null : new String(rawdata, utf8); - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Got vote data from path " + path + + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Got vote data from path " + path + " with code " + code + " and data " + data); int index = getIndex(path); synchronized (nextMasterData) { if (code != KeeperException.Code.OK.intValue()) { if (code == KeeperException.Code.NONODE.intValue()) { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Node at " + path + + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Node at " + path + " removed, got no other option than counting it as down."); } else { log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failure code " + code + @@ -129,7 +129,7 @@ public class MasterDataGatherer { if (value.equals(nextMasterData.get(index))) { log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + ", which already was " + value + "."); } else { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + ". Altering vote from " + nextMasterData.get(index) + " to " + value + "."); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + ". Altering vote from " + nextMasterData.get(index) + " to " + value + "."); nextMasterData.put(index, value); } } else { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java index 834ecab9f5b..7e878c80fdb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java @@ -58,17 +58,17 @@ public class ZooKeeperDatabase extends Database { Event.KeeperState newState = watchedEvent.getState(); if (state == null || !state.equals(newState)) switch (newState) { case Expired: - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Zookeeper session expired"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Zookeeper session expired"); sessionOpen = false; listener.handleZooKeeperSessionDown(); break; case Disconnected: - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Lost connection to zookeeper server"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Lost connection to zookeeper server"); sessionOpen = false; listener.handleZooKeeperSessionDown(); break; case SyncConnected: - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Connection to zookeeper server established. Refetching master data"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Connection to zookeeper server established. Refetching master data"); if (masterDataGatherer != null) { masterDataGatherer.restart(); } @@ -139,14 +139,14 @@ public class ZooKeeperDatabase extends Database { createNode(zooKeeperRoot, "published_state_bundle", new byte[0]); // TODO dedupe string constants byte val[] = String.valueOf(nodeIndex).getBytes(utf8); deleteNodeIfExists(getMyIndexPath()); - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Creating ephemeral master vote node with vote to self."); session.create(getMyIndexPath(), val, acl, CreateMode.EPHEMERAL); } private void deleteNodeIfExists(String path) throws KeeperException, InterruptedException { if (session.exists(path, false) != null) { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Removing master vote node."); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Removing master vote node."); session.delete(path, -1); } } @@ -187,7 +187,7 @@ public class ZooKeeperDatabase extends Database { byte val[] = String.valueOf(wantedMasterIndex).getBytes(utf8); try{ session.setData(getMyIndexPath(), val, -1); - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Stored new vote in ephemeral node. " + nodeIndex + " -> " + wantedMasterIndex); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Stored new vote in ephemeral node. " + nodeIndex + " -> " + wantedMasterIndex); return true; } catch (InterruptedException e) { throw (InterruptedException) new InterruptedException("Interrupted").initCause(e); @@ -199,7 +199,7 @@ public class ZooKeeperDatabase extends Database { public boolean storeLatestSystemStateVersion(int version) throws InterruptedException { byte data[] = Integer.toString(version).getBytes(utf8); try{ - log.log(LogLevel.INFO, String.format("Fleetcontroller %d: Storing new cluster state version in ZooKeeper: %d", nodeIndex, version)); + log.log(Level.INFO, String.format("Fleetcontroller %d: Storing new cluster state version in ZooKeeper: %d", nodeIndex, version)); var stat = session.setData(zooKeeperRoot + "latestversion", data, lastKnownStateVersionZNodeVersion); lastKnownStateVersionZNodeVersion = stat.getVersion(); return true; @@ -222,7 +222,7 @@ public class ZooKeeperDatabase extends Database { byte[] data = session.getData(zooKeeperRoot + "latestversion", false, stat); lastKnownStateVersionZNodeVersion = stat.getVersion(); final Integer versionNumber = Integer.valueOf(new String(data, utf8)); - log.log(LogLevel.INFO, String.format("Fleetcontroller %d: Read cluster state version %d from ZooKeeper " + + log.log(Level.INFO, String.format("Fleetcontroller %d: Read cluster state version %d from ZooKeeper " + "(znode version %d)", nodeIndex, versionNumber, stat.getVersion())); return versionNumber; } catch (InterruptedException e) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index e3ae95b0abe..c12f906fe09 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -112,7 +112,7 @@ public class RpcServer { SlobrokList slist = new SlobrokList(); slist.setup(slobrokConnectionSpecs); Spec spec = new Spec(HostName.getLocalhost(), acceptor.port()); - log.log(LogLevel.INFO, "Registering " + spec + " with slobrok at " + slobroks); + log.log(Level.INFO, "Registering " + spec + " with slobrok at " + slobroks); if (slobrokBackOffPolicy != null) { register = new Register(supervisor, slist, spec, slobrokBackOffPolicy); } else { @@ -301,7 +301,7 @@ public class RpcServer { req.returnValues().add(new StringValue(message)); req.returnRequest(); if (nodeState.getState() == State.UP && node.getPrematureCrashCount() > 0) { - log.log(LogLevel.INFO, "Clearing premature crash count of " + node.getPrematureCrashCount() + " as wanted state was set to up"); + log.log(Level.INFO, "Clearing premature crash count of " + node.getPrematureCrashCount() + " as wanted state was set to up"); node.setPrematureCrashCount(0); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index 19181f2d1eb..8b3bec1b2cf 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -83,7 +83,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { Thread.sleep(10); } if (!isConnected()) { - log.log(LogLevel.INFO, "Fleetcontroller: Server Socket not ready after connect()"); + log.log(Level.INFO, "Fleetcontroller: Server Socket not ready after connect()"); } log.log(Level.FINE, "Fleet controller status page viewer listening to " + ssocket.getLocalSocketAddress()); monitor.notifyAll(); @@ -102,7 +102,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { public void setPort(int port) throws java.io.IOException, InterruptedException { // Only bother to reconnect if we were connected to begin with, we care about what port it runs on, and it's not already running there if (port != 0 && isConnected() && port != ((InetSocketAddress) ssocket.getLocalSocketAddress()).getPort()) { - log.log(LogLevel.INFO, "Exchanging port used by status server. Moving from port " + log.log(Level.INFO, "Exchanging port used by status server. Moving from port " + ((InetSocketAddress) ssocket.getLocalSocketAddress()).getPort() + " to port " + port); disconnect(); this.port = port; @@ -250,7 +250,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { if (connection != null) try{ connection.close(); } catch (IOException e) { - log.log(LogLevel.INFO, "Failed to close socket " + connection + ": " + e.getMessage()); + log.log(Level.INFO, "Failed to close socket " + connection + ": " + e.getMessage()); } } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index 6d8afa14f3e..fa2eb80506e 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -299,7 +299,7 @@ public abstract class FleetControllerTest implements Waiter { protected void tearDownSystem() throws Exception { if (testName != null) { - //log.log(LogLevel.INFO, "STOPPING TEST " + testName); + //log.log(Level.INFO, "STOPPING TEST " + testName); System.err.println("STOPPING TEST " + testName); testName = null; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index c7f25864341..557765ca761 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -119,18 +119,18 @@ public class MasterElectionTest extends FleetControllerTest { @Ignore public void testMasterElection() throws Exception { startingTest("MasterElectionTest::testMasterElection"); - log.log(LogLevel.INFO, "STARTING TEST: MasterElectionTest::testMasterElection()"); + log.log(Level.INFO, "STARTING TEST: MasterElectionTest::testMasterElection()"); FleetControllerOptions options = defaultOptions("mycluster"); options.masterZooKeeperCooldownPeriod = 1; setUpFleetController(5, false, options); waitForMaster(0); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); fleetControllers.get(0).shutdown(); waitForMaster(1); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 1"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 1"); fleetControllers.get(1).shutdown(); waitForMaster(2); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 2"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 2"); fleetControllers.get(2).shutdown(); // Too few for there to be a master at this point @@ -139,23 +139,23 @@ public class MasterElectionTest extends FleetControllerTest { assertFalse("Fleet controller " + i, fleetControllers.get(i).isMaster()); } - log.log(LogLevel.INFO, "STARTING FLEET CONTROLLER 2"); + log.log(Level.INFO, "STARTING FLEET CONTROLLER 2"); fleetControllers.set(2, createFleetController(usingFakeTimer, fleetControllers.get(2).getOptions(), true, null)); waitForMaster(2); - log.log(LogLevel.INFO, "STARTING FLEET CONTROLLER 0"); + log.log(Level.INFO, "STARTING FLEET CONTROLLER 0"); fleetControllers.set(0, createFleetController(usingFakeTimer, fleetControllers.get(0).getOptions(), true, null)); waitForMaster(0); - log.log(LogLevel.INFO, "STARTING FLEET CONTROLLER 1"); + log.log(Level.INFO, "STARTING FLEET CONTROLLER 1"); fleetControllers.set(1, createFleetController(usingFakeTimer, fleetControllers.get(1).getOptions(), true, null)); waitForMaster(0); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 4"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 4"); fleetControllers.get(4).shutdown(); waitForMaster(0); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 3"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 3"); fleetControllers.get(3).shutdown(); waitForMaster(0); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 2"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 2"); fleetControllers.get(2).shutdown(); // Too few for there to be a master at this point @@ -166,20 +166,20 @@ public class MasterElectionTest extends FleetControllerTest { } private void waitForMaster(int master) { - log.log(LogLevel.INFO, "Entering waitForMaster"); + log.log(Level.INFO, "Entering waitForMaster"); boolean isOnlyMaster = false; for (int i=0; i < FleetControllerTest.timeoutMS; i+=100) { if (!fleetControllers.get(master).isMaster()) { - log.log(LogLevel.INFO, "Node " + master + " is not master yet, sleeping more"); + log.log(Level.INFO, "Node " + master + " is not master yet, sleeping more"); timer.advanceTime(100); waitForCompleteCycle(master); } else { - log.log(LogLevel.INFO, "Node " + master + " is master. Checking that noone else is master"); + log.log(Level.INFO, "Node " + master + " is master. Checking that noone else is master"); isOnlyMaster = true; for (int j=0; j "Created connection to config source at " + spec.toString()); return; } else { - log.log(LogLevel.INFO, "Could not connect to config source at " + spec.toString()); + log.log(Level.INFO, "Could not connect to config source at " + spec.toString()); } target.close(); } String extra = ""; - log.log(LogLevel.INFO, "Could not connect to any config source in set " + configSourceSet.toString() + + log.log(Level.INFO, "Could not connect to any config source in set " + configSourceSet.toString() + ", please make sure config server(s) are running. " + extra); } } @@ -165,7 +165,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { activeSubscribers.put(configCacheKey, subscriber); exec.execute(subscriber); } catch (ConfigurationRuntimeException e) { - log.log(LogLevel.INFO, "Subscribe for '" + configCacheKey + "' failed, closing subscriber"); + log.log(Level.INFO, "Subscribe for '" + configCacheKey + "' failed, closing subscriber"); subscriber.cancel(); } } @@ -235,7 +235,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { log.log(Level.FINE, () -> "Call returnOkResponse for " + config.getKey() + "," + config.getGeneration()); rpcServer.returnOkResponse(request, config); } else { - log.log(LogLevel.INFO, "Could not remove " + config.getKey() + " from delayedResponses queue, already removed"); + log.log(Level.INFO, "Could not remove " + config.getKey() + " from delayedResponses queue, already removed"); } } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java index 146a5e644ef..ffe7f7bec46 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java @@ -70,7 +70,7 @@ class CachedFilesMaintainer implements Runnable { .filter(fileReference -> isFileLastModifiedBefore(new File(directory, fileReference), deleteNotUsedSinceInstant)) .collect(Collectors.toSet()); if (filesToDelete.size() > 0) { - log.log(LogLevel.INFO, "Files that can be deleted in " + directory + " (not used since " + deleteNotUsedSinceInstant + "): " + filesToDelete); + log.log(Level.INFO, "Files that can be deleted in " + directory + " (not used since " + deleteNotUsedSinceInstant + "): " + filesToDelete); filesToDelete.forEach(fileReference -> { File file = new File(directory, fileReference); if (!IOUtils.recursiveDeleteDir(file)) diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java index 3b8a882963c..a25e86926a1 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java @@ -122,7 +122,7 @@ class FileDistributionRpcServer { req.returnValues().add(new StringValue(file.get().getAbsolutePath())); log.log(Level.FINE, () -> "File reference '" + fileReference.value() + "' available at " + file.get()); } else { - log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); + log.log(Level.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found"); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java index 74393ca07f9..21baf873f6d 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java @@ -79,11 +79,11 @@ class UrlDownloadRpcServer { setIfModifiedSince(connection, downloadDir); // don't download if we already have the file if (connection.getResponseCode() == 200) { - log.log(LogLevel.INFO, "Downloading URL '" + url + "'"); + log.log(Level.INFO, "Downloading URL '" + url + "'"); downloadFile(req, connection, downloadDir); } else if (connection.getResponseCode() == 304) { - log.log(LogLevel.INFO, "URL '" + url + "' already downloaded (server response: 304)"); + log.log(Level.INFO, "URL '" + url + "' already downloaded (server response: 304)"); req.returnValues().add(new StringValue(new File(downloadDir, CONTENTS_FILE_NAME).getAbsolutePath())); } else { @@ -112,7 +112,7 @@ class UrlDownloadRpcServer { new RequestTracker().trackRequest(downloadDir); req.returnValues().add(new StringValue(contentsPath.getAbsolutePath())); log.log(Level.FINE, () -> "URL '" + url + "' available at " + contentsPath); - log.log(LogLevel.INFO, String.format("Download of URL '%s' done in %.3f seconds", + log.log(Level.INFO, String.format("Download of URL '%s' done in %.3f seconds", url, (System.currentTimeMillis() -start) / 1000.0)); } else { log.log(LogLevel.ERROR, "Downloaded URL '" + url + "' not found, returning error"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index be607907854..583d560994e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -215,7 +215,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye try (ActionTimer timer = timerFor(applicationId, "deployment.prepareMillis")) { ConfigChangeActions actions = session.prepare(logger, prepareParams, currentActiveApplicationSet, tenant.getPath(), now); logConfigChangeActions(actions, logger); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); return new PrepareResult(sessionId, actions, deployLog); } } @@ -380,17 +380,17 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye remoteSession = getRemoteSession(tenant, sessionId); Transaction deleteTransaction = remoteSession.createDeleteTransaction(); deleteTransaction.commit(); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); if ( ! waitTime.isZero() && localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); } else { deleteTransaction.rollbackOrLog(); throw new InternalServerException(applicationId + " was not deleted (waited " + waitTime + "), session " + sessionId); } } catch (NotFoundException e) { // For the case where waiting timed out in a previous attempt at deleting the application, continue and do the steps below - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Active session exists, but has not been deleted properly. Trying to cleanup"); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Active session exists, but has not been deleted properly. Trying to cleanup"); } NestedTransaction transaction = new NestedTransaction(); @@ -400,7 +400,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye transaction.add(tenantApplications.createDeleteTransaction(applicationId)); hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); - transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId)); + transaction.onCommitted(() -> log.log(Level.INFO, "Deleted " + applicationId)); transaction.commit(); return true; } @@ -466,7 +466,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .filter(fileReference -> isFileLastModifiedBefore(new File(fileReferencesPath, fileReference), instant)) .collect(Collectors.toSet()); if (fileReferencesToDelete.size() > 0) { - log.log(LogLevel.INFO, "Will delete file references not in use: " + fileReferencesToDelete); + log.log(Level.INFO, "Will delete file references not in use: " + fileReferencesToDelete); fileReferencesToDelete.forEach(fileReference -> { File file = new File(fileReferencesPath, fileReference); if ( ! IOUtils.recursiveDeleteDir(file)) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 98c751dcee5..35cbd0c4303 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -118,7 +118,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable @Override public void deconstruct() { - log.log(LogLevel.INFO, "Stopping config server"); + log.log(Level.INFO, "Stopping config server"); down(); server.stop(); log.log(Level.FINE, "RPC server stopped"); @@ -148,7 +148,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable public void start() { if (versionState.isUpgraded()) { - log.log(LogLevel.INFO, "Config server upgrading from " + versionState.storedVersion() + " to " + log.log(Level.INFO, "Config server upgrading from " + versionState.storedVersion() + " to " + versionState.currentVersion() + ". Redeploying all applications"); try { if ( ! redeployAllApplications()) { @@ -156,7 +156,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable return; // Status will not be set to 'up' since we return here } versionState.saveNewVersion(); - log.log(LogLevel.INFO, "All applications redeployed successfully"); + log.log(Level.INFO, "All applications redeployed successfully"); } catch (Exception e) { log.log(LogLevel.ERROR, "Redeployment of applications failed", e); redeployingApplicationsFailed(); @@ -211,7 +211,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable do { applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed); if ( ! applicationsNotRedeployed.isEmpty()) { - log.log(LogLevel.INFO, "Redeployment of " + applicationsNotRedeployed + + log.log(Level.INFO, "Redeployment of " + applicationsNotRedeployed + " failed, will retry in " + sleepTimeWhenRedeployingFails); Thread.sleep(sleepTimeWhenRedeployingFails.toMillis()); } @@ -243,7 +243,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable } catch (ExecutionException e) { ApplicationId app = f.getKey(); if (e.getCause() instanceof TransientException) { - log.log(LogLevel.INFO, "Redeploying " + app + + log.log(Level.INFO, "Redeploying " + app + " failed with transient error, will retry after bootstrap: " + Exceptions.toMessageString(e)); } else { log.log(LogLevel.WARNING, "Redeploying " + app + " failed, will retry", e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index bc022a3342c..db10d9dcc73 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -127,7 +127,7 @@ public class SuperModelManager implements SuperModelProvider { public void markAsComplete() { // Invoked on component graph bootstrap (even before ConfigServerBootstrap), // there is no need to bump generation counter. - logger.log(LogLevel.INFO, "Super model is complete"); + logger.log(Level.INFO, "Super model is complete"); SuperModel newSuperModel = getSuperModel().cloneAsComplete(); superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource); listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 318cd367e30..4f04724a0a8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -171,7 +171,7 @@ public class TenantApplications { private void applicationRemoved(ApplicationId applicationId) { reloadHandler.removeApplication(applicationId); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Application removed: " + applicationId); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Application removed: " + applicationId); } private void applicationAdded(ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 37dd0502ee8..ede777265b7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -149,7 +149,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { session.waitUntilActivated(timeoutBudget); - log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() + + log.log(Level.INFO, session.logPre() + "Session " + session.getSessionId() + " activated successfully using " + (hostProvisioner.isPresent() ? hostProvisioner.get().getClass().getSimpleName() : "no host provisioner") + ". Config generation " + session.getMetaData().getGeneration() + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java index 52fe57a7923..7dc6b5185a6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java @@ -33,7 +33,7 @@ public class HttpErrorResponse extends HttpResponse { root.setString("error-code", errorType); root.setString("message", msg); if (code != 200) { - log.log(LogLevel.INFO, "Returning response with response code " + code + ", error-code:" + errorType + ", message=" + msg); + log.log(Level.INFO, "Returning response with response code " + code + ", error-code:" + errorType + ", message=" + msg); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 361153d2817..1aaa090d145 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -118,7 +118,7 @@ public abstract class ModelsBuilder { } catch (RuntimeException e) { if (shouldSkipCreatingMajorVersionOnError(majorVersions, majorVersion)) { - log.log(LogLevel.INFO, applicationId + ": Skipping major version " + majorVersion, e); + log.log(Level.INFO, applicationId + ": Skipping major version " + majorVersion, e); } else { if (e instanceof NullPointerException || e instanceof NoSuchElementException | e instanceof UncheckedTimeoutException) { log.log(LogLevel.WARNING, "Unexpected error when building model ", e); @@ -197,7 +197,7 @@ public abstract class ModelsBuilder { // allow failure to create old config models if there is a validation override that allow skipping old // config models (which is always true for manually deployed zones) if (allApplicationVersions.size() > 0 && allApplicationVersions.get(0).getModel().skipOldConfigModels(now)) - log.log(LogLevel.INFO, applicationId + ": Skipping old version (due to validation override)"); + log.log(Level.INFO, applicationId + ": Skipping old version (due to validation override)"); else { log.log(LogLevel.ERROR, applicationId + ": Failed to build version " + version); throw e; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index e5918d19bc0..977e1f8cc28 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -182,7 +182,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Override public void run() { - log.log(LogLevel.INFO, "Rpc server will listen on port " + spec.port()); + log.log(Level.INFO, "Rpc server will listen on port " + spec.port()); try { Acceptor acceptor = supervisor.listen(spec); isRunning = true; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java index 4484fa55057..3dbb3fd11e3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -156,7 +156,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { boolean isAuthorizationException = throwable instanceof AuthorizationException; String errorMessage = String.format("For request '%s' from '%s': %s", request.methodName(), request.target().toString(), throwable.getMessage()); if (!isAuthorizationException || ((AuthorizationException) throwable).type() != Type.SILENT) { - log.log(LogLevel.INFO, errorMessage); + log.log(Level.INFO, errorMessage); } log.log(Level.FINE, throwable, throwable::getMessage); JrtErrorCode error = isAuthorizationException ? JrtErrorCode.UNAUTHORIZED : JrtErrorCode.AUTHORIZATION_FAILED; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 0f96d70a3b1..9da3ad2fd7a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -94,7 +94,7 @@ public class RemoteSession extends Session { reloadHandler.reloadConfig(app); log.log(Level.FINE, () -> logPre() + "Notifying " + waiter); notifyCompletion(waiter); - log.log(LogLevel.INFO, logPre() + "Session activated: " + getSessionId()); + log.log(Level.INFO, logPre() + "Session activated: " + getSessionId()); } @Override @@ -124,7 +124,7 @@ public class RemoteSession extends Session { if (e.getCause().getClass() != KeeperException.NoNodeException.class) { throw e; } else { - log.log(LogLevel.INFO, "Not able to notify completion for session: " + getSessionId() + ", node has been deleted"); + log.log(Level.INFO, "Not able to notify completion for session: " + getSessionId() + ", node has been deleted"); } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index e4e097cf448..d66525326a2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -88,7 +88,7 @@ public class RemoteSessionRepo extends SessionRepo { if (session == null) continue; // Internal sessions not in synch with zk, continue Instant created = Instant.ofEpochSecond(session.getCreateTime()); if (sessionHasExpired(created, expiryTime)) { - log.log(LogLevel.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); + log.log(Level.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); session.delete(); deleted++; } @@ -166,7 +166,7 @@ public class RemoteSessionRepo extends SessionRepo { if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { log.log(Level.FINE, () -> "Found active application for session " + session.getSessionId() + " , loading it"); reloadHandler.reloadConfig(session.ensureApplicationLoaded()); - log.log(LogLevel.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); + log.log(Level.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); return; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index fd32cb020d6..1a366063bae 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -82,7 +82,7 @@ public class SessionZooKeeperClient { String data = configCurator.getData(sessionStatusPath.getAbsolute()); return Session.Status.parse(data); } catch (Exception e) { - log.log(LogLevel.INFO, "Unable to read session status, assuming it was deleted"); + log.log(Level.INFO, "Unable to read session status, assuming it was deleted"); return Session.Status.NONE; } } @@ -134,7 +134,7 @@ public class SessionZooKeeperClient { transaction.add(curatorTransaction); transaction.commit(); } catch (RuntimeException e) { - log.log(LogLevel.INFO, "Error deleting session (" + sessionPath.getAbsolute() + ") from zookeeper", e); + log.log(Level.INFO, "Error deleting session (" + sessionPath.getAbsolute() + ") from zookeeper", e); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java index a1de394f61a..b249211081f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java @@ -32,7 +32,7 @@ public class TenantDebugger implements TreeCacheListener { case NODE_ADDED: case NODE_REMOVED: case NODE_UPDATED: - log.log(LogLevel.INFO, event.toString()); + log.log(Level.INFO, event.toString()); break; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 45b22ae156c..49bfc20233e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -203,7 +203,7 @@ public class TenantRepository { TenantName tenantName = builder.getTenantName(); if (tenants.containsKey(tenantName)) return; - log.log(LogLevel.INFO, "Creating tenant '" + tenantName + "'"); + log.log(Level.INFO, "Creating tenant '" + tenantName + "'"); Tenant tenant = builder.build(); notifyNewTenant(tenant); tenants.putIfAbsent(tenantName, tenant); @@ -280,7 +280,7 @@ public class TenantRepository { if ( ! tenants.containsKey(name)) throw new IllegalArgumentException("Deleting '" + name + "' failed, tenant does not exist"); - log.log(LogLevel.INFO, "Deleting tenant '" + name + "'"); + log.log(Level.INFO, "Deleting tenant '" + name + "'"); tenants.get(name).delete(); } @@ -289,7 +289,7 @@ public class TenantRepository { if (tenant == null) throw new IllegalArgumentException("Closing '" + name + "' failed, tenant does not exist"); - log.log(LogLevel.INFO, "Closing tenant '" + name + "'"); + log.log(Level.INFO, "Closing tenant '" + name + "'"); notifyRemovedTenant(name); tenant.close(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java index b3d783633a5..fbfbe11cdc6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java @@ -138,7 +138,7 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host public void removeApplicationsExcept(Set applications) { for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { if ( ! applications.contains(activeApplication)) { - log.log(LogLevel.INFO, "Will remove deleted application " + activeApplication.toShortString()); + log.log(Level.INFO, "Will remove deleted application " + activeApplication.toShortString()); removeApplication(activeApplication); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index 29691668975..78b65622150 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -78,7 +78,7 @@ public class StateMonitor extends AbstractComponent { public void status(Status status) { if (status != this.status) { - log.log(LogLevel.INFO, "Changing health status code from '" + this.status + "' to '" + status.name() + "'"); + log.log(Level.INFO, "Changing health status code from '" + this.status + "' to '" + status.name() + "'"); this.status = status; } } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 81a2c8a9373..f7e03a2098e 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -109,7 +109,7 @@ public final class ConfiguredApplication implements Application { static { LogSetup.initVespaLogging("Container"); - log.log(LogLevel.INFO, "Starting container"); + log.log(Level.INFO, "Starting container"); } /** @@ -177,7 +177,7 @@ public final class ConfiguredApplication implements Application { Spec mySpec = new Spec(HostName.getLocalhost(), acceptor.port()); slobrokRegistrator = new Register(supervisor, slobrokList, mySpec); slobrokRegistrator.registerName(qrConfig.rpc().slobrokId()); - log.log(LogLevel.INFO, "Registered name '" + qrConfig.rpc().slobrokId() + + log.log(Level.INFO, "Registered name '" + qrConfig.rpc().slobrokId() + "' at " + mySpec + " with: " + slobrokList); return slobrokRegistrator; } diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java index 5da472b9b28..a609ee73a97 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java @@ -48,7 +48,7 @@ public class MbusServerProvider implements Provider { @Override public void deconstruct() { - log.log(LogLevel.INFO, "Deconstructing mbus server: " + server); + log.log(Level.INFO, "Deconstructing mbus server: " + server); server.close(); server.release(); sessionRef.getReference().close(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index 8cef88cacf0..a1b97dad56a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -86,7 +86,7 @@ public class EndpointCertificateManager { try { this.backfillCertificateMetadata(); } catch (Throwable t) { - log.log(LogLevel.INFO, "Unexpected Throwable caught while backfilling certificate metadata", t); + log.log(Level.INFO, "Unexpected Throwable caught while backfilling certificate metadata", t); } }, 1, 10, TimeUnit.MINUTES); } @@ -95,7 +95,7 @@ public class EndpointCertificateManager { var t0 = Instant.now(); Optional metadata = getOrProvision(instance, zone); Duration duration = Duration.between(t0, Instant.now()); - if (duration.toSeconds() > 30) log.log(LogLevel.INFO, String.format("Getting endpoint certificate metadata for %s took %d seconds!", instance.id().serializedForm(), duration.toSeconds())); + if (duration.toSeconds() > 30) log.log(Level.INFO, String.format("Getting endpoint certificate metadata for %s took %d seconds!", instance.id().serializedForm(), duration.toSeconds())); return metadata; } @@ -175,7 +175,7 @@ public class EndpointCertificateManager { EndpointCertificateMetadata providerMetadata = sanToEndpointCertificate.get(hashedCn); if (providerMetadata == null) { - log.log(LogLevel.INFO, "No matching certificate provider metadata found for application " + applicationId.serializedForm()); + log.log(Level.INFO, "No matching certificate provider metadata found for application " + applicationId.serializedForm()); return; } @@ -189,7 +189,7 @@ public class EndpointCertificateManager { providerMetadata.issuer()); if (mode == BackfillMode.DRYRUN) { - log.log(LogLevel.INFO, "Would update stored metadata " + storedMetaData + " with data from provider: " + backfilledMetadata); + log.log(Level.INFO, "Would update stored metadata " + storedMetaData + " with data from provider: " + backfilledMetadata); } else if (mode == BackfillMode.ENABLE) { curator.writeEndpointCertificateMetadata(applicationId, backfilledMetadata); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index 75a57ba49e5..6f60106c79e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -78,7 +78,7 @@ public class NameServiceForwarder { "requests. This likely means that the name service is not successfully " + "executing requests"); } - log.log(LogLevel.INFO, "Queueing name service request: " + request); + log.log(Level.INFO, "Queueing name service request: " + request); db.writeNameServiceQueue(queue.with(request, priority).last(maxQueuedRequests)); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java index f1b8d40cf80..e9ff039f8ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java @@ -47,7 +47,7 @@ public class NameServiceDispatcher extends Maintainer { var dispatched = queue.last(requestCount); if (!dispatched.requests().isEmpty()) { - log.log(LogLevel.INFO, "Dispatched name service request(s) in " + + log.log(Level.INFO, "Dispatched name service request(s) in " + Duration.between(instant, clock.instant()) + ": " + dispatched.requests()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 8eccb92c2e9..48118087a54 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -70,7 +70,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { } } catch (Exception e) { - logger.log(LogLevel.INFO, () -> "Exception mapping Athenz principal to roles: " + Exceptions.toMessageString(e)); + logger.log(Level.INFO, () -> "Exception mapping Athenz principal to roles: " + Exceptions.toMessageString(e)); } return Optional.empty(); } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 49a101a2ed6..3a92d005b79 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -78,7 +78,7 @@ public class DockerImpl implements Docker { scheduledPulls.add(image); - logger.log(LogLevel.INFO, "Starting download of " + image.asString()); + logger.log(Level.INFO, "Starting download of " + image.asString()); dockerClient.pullImageCmd(image.asString()).exec(new ImagePullCallback(image)); return true; @@ -352,7 +352,7 @@ public class DockerImpl implements Docker { @Override public void onComplete() { if (imageIsDownloaded(dockerImage)) { - logger.log(LogLevel.INFO, "Download completed: " + dockerImage.asString()); + logger.log(Level.INFO, "Download completed: " + dockerImage.asString()); removeScheduledPoll(dockerImage); } else { numberOfDockerApiFails.increment(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 79df0db0d17..493871d45c0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -317,7 +317,7 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { @Override public void handleReply(Reply reply) { if (reply.getTrace().getLevel() > 0) { - log.log(LogLevel.INFO, reply.getTrace().toString()); + log.log(Level.INFO, reply.getTrace().toString()); } Response response = toResponse(reply); if (handler != null) { diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java index e95eaa754b6..32c381c1e38 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java @@ -160,7 +160,7 @@ class FileAcquirerImpl implements FileAcquirer { } else if (!request.isError()) { throw new RuntimeException("Invalid response: " + request.returnValues()); } else if (temporaryError(request.errorCode())) { - log.log(LogLevel.INFO, "Retrying waitFor for " + fileReference + ": " + request.errorCode() + " -- " + request.errorMessage()); + log.log(Level.INFO, "Retrying waitFor for " + fileReference + ": " + request.errorCode() + " -- " + request.errorMessage()); Thread.sleep(1000); } else { if (request.errorCode() == FileDistributionErrorCode.fileReferenceDoesNotExists) diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 2db66f6c859..4af10b69be6 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -192,7 +192,7 @@ public class FileReceiver { } catch (FileAlreadyExistsException e) { // Don't fail if it already exists (we might get the file from several config servers when retrying, servers are down etc. // so it might be written already). Delete temp file/dir in that case, to avoid filling the disk. - log.log(LogLevel.INFO, "Failed moving file '" + tempFile.getAbsolutePath() + "' to '" + destination.getAbsolutePath() + + log.log(Level.INFO, "Failed moving file '" + tempFile.getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "', '" + destination.getAbsolutePath() + "' already exists"); deleteFileOrDirectory(tempFile); } catch (IOException e) { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java index 0873ed57adb..4e0d5e4e32a 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java @@ -49,19 +49,19 @@ public class RpcTester { e.printStackTrace(); } - log.log(LogLevel.INFO, "Read blob from " + file.getAbsolutePath()); + log.log(Level.INFO, "Read blob from " + file.getAbsolutePath()); Supervisor supervisor = new Supervisor(new Transport()); Spec spec = new Spec("tcp/localhost:19090"); - log.log(LogLevel.INFO, "Connecting to " + spec); + log.log(Level.INFO, "Connecting to " + spec); Target target = supervisor.connect(spec); if (! target.isValid()) { - log.log(LogLevel.INFO, "Could not connect"); + log.log(Level.INFO, "Could not connect"); System.exit(1); } else { - log.log(LogLevel.INFO, "Connected to " + spec); + log.log(Level.INFO, "Connected to " + spec); } new RpcTester(target).call(fileReference, filename, blob); @@ -77,12 +77,12 @@ public class RpcTester { void receive(FileReference reference, String filename, byte[] content) { - log.log(LogLevel.INFO, "Preparing receive call for " + reference.value() + " and file " + filename); + log.log(Level.INFO, "Preparing receive call for " + reference.value() + " and file " + filename); XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); Request fileBlob = new Request("filedistribution.receiveFile"); - log.log(LogLevel.INFO, "Calling " + fileBlob.methodName() + " with target " + target); + log.log(Level.INFO, "Calling " + fileBlob.methodName() + " with target " + target); fileBlob.parameters().add(new StringValue(reference.value())); fileBlob.parameters().add(new StringValue(filename)); @@ -90,9 +90,9 @@ public class RpcTester { fileBlob.parameters().add(new Int64Value(hasher.hash(ByteBuffer.wrap(content), 0))); fileBlob.parameters().add(new Int32Value(0)); fileBlob.parameters().add(new StringValue("OK")); - log.log(LogLevel.INFO, "Doing invokeSync"); + log.log(Level.INFO, "Doing invokeSync"); target.invokeSync(fileBlob, 5); - log.log(LogLevel.INFO, "Done with invokeSync"); + log.log(Level.INFO, "Done with invokeSync"); } } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java index 0082950ffb0..6976654a744 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java @@ -70,12 +70,12 @@ public class FlagDbFile { FlagData existingFlagData = currentFlagData.get(data.id()); if (existingFlagData == null) { - logger.log(LogLevel.INFO, "New flag " + data.id() + ": " + data.serializeToJson()); + logger.log(Level.INFO, "New flag " + data.id() + ": " + data.serializeToJson()); modified = true; // Could also consider testing with FlagData::equals, but that would be too fragile? } else if (!Objects.equals(data.serializeToJson(), existingFlagData.serializeToJson())){ - logger.log(LogLevel.INFO, "Updating flag " + data.id() + " from " + + logger.log(Level.INFO, "Updating flag " + data.id() + " from " + existingFlagData.serializeToJson() + " to " + data.serializeToJson()); modified = true; } @@ -83,7 +83,7 @@ public class FlagDbFile { if (!flagIdsToBeRemoved.isEmpty()) { String flagIdsString = flagIdsToBeRemoved.stream().map(FlagId::toString).collect(Collectors.joining(", ")); - logger.log(LogLevel.INFO, "Removing flags " + flagIdsString); + logger.log(Level.INFO, "Removing flags " + flagIdsString); modified = true; } diff --git a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java index 580c621cb20..4988357ee8c 100644 --- a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java @@ -34,7 +34,7 @@ public abstract class AbstractPluginLoader implements PluginLoader { String enable = config.get("enable", "true"); if (! enable.equals("true")) { - log.log(LogLevel.INFO, pname + ": plugin disabled by config"); + log.log(Level.INFO, pname + ": plugin disabled by config"); return; } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 8524d7e8c06..feec51643c4 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -435,7 +435,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, RoutingTableSpec table = spec.getTable(i); String name = table.getProtocol(); if (!protocolRepository.hasProtocol(name)) { - log.log(LogLevel.INFO, "Protocol '" + name + "' is not supported, ignoring routing table."); + log.log(Level.INFO, "Protocol '" + name + "' is not supported, ignoring routing table."); continue; } tables.put(name, new RoutingTable(table)); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java index b8faafe54bb..c1029175080 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java @@ -144,7 +144,7 @@ public class RPCTarget implements RequestWaiter { log.log(LogLevel.WARNING, "Failed to parse '" + str + "' as version for target '" + name + "'.", e); } } else { - log.log(LogLevel.INFO, "Method mbus.getVersion() failed for target '" + name + "'; " + + log.log(Level.INFO, "Method mbus.getVersion() failed for target '" + name + "'; " + req.errorMessage()); } handlers = versionHandlers; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index 0a17810d3cc..905682df889 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -64,7 +64,7 @@ public abstract class HttpMetricFetcher { private void logMessage(String message, int timesFetched) { if (service.isAlive() && timesFetched > 5) { - log.log(LogLevel.INFO, message); + log.log(Level.INFO, message); } else { log.log(Level.FINE, message); } @@ -74,7 +74,7 @@ public abstract class HttpMetricFetcher { if (timesFetched > 5) { log.log(LogLevel.WARNING, message); } else { - log.log(LogLevel.INFO, message); + log.log(Level.INFO, message); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 6983ff7576d..b58fd3f67f1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -63,7 +63,7 @@ public abstract class ApplicationMaintainer extends Maintainer { */ protected void deploy(ApplicationId application) { if (pendingDeployments.addIfAbsent(application)) { // Avoid queuing multiple deployments for same application - log.log(LogLevel.INFO, application + " will be deployed, last deploy time " + + log.log(Level.INFO, application + " will be deployed, last deploy time " + getLastDeployTime(application)); deploymentExecutor.execute(() -> deployWithLock(application)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java index 220387bd157..932c435221c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -29,7 +29,7 @@ public class InfrastructureProvisioner extends Maintainer { try { infraDeployer.activateAllSupportedInfraApplications(true); } catch (RuntimeException e) { - logger.log(LogLevel.INFO, "Failed to deploy supported infrastructure applications, " + + logger.log(Level.INFO, "Failed to deploy supported infrastructure applications, " + "will sleep 30s before propagating failure, to allow inspection of zk", e.getMessage()); try { Thread.sleep(30_000); } catch (InterruptedException ignored) { } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java index 2e7706d5701..6145a696af8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java @@ -83,7 +83,7 @@ class MaintenanceDeployment implements Closeable { return true; } catch (TransientException e) { metric.add("maintenanceDeployment.transientFailure", 1, metric.createContext(Map.of())); - log.log(LogLevel.INFO, "Failed to maintenance deploy " + application + " with a transient error: " + + log.log(Level.INFO, "Failed to maintenance deploy " + application + " with a transient error: " + Exceptions.toMessageString(e)); return false; } catch (RuntimeException e) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 7eccbe8366e..cffbfec8f4b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -366,7 +366,7 @@ public class NodeFailer extends Maintainer { deployment.get().activate(); return true; } catch (TransientException e) { - log.log(LogLevel.INFO, "Failed to redeploy " + node.allocation().get().owner() + + log.log(Level.INFO, "Failed to redeploy " + node.allocation().get().owner() + " with a transient error, will be retried by application maintainer: " + Exceptions.toMessageString(e)); return true; } catch (RuntimeException e) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 845b4a462c0..347c66d5426 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -124,7 +124,7 @@ public class CuratorDatabaseClient { transaction.commit(); for (Node node : nodes) - log.log(LogLevel.INFO, "Added " + node); + log.log(Level.INFO, "Added " + node); return nodes; } @@ -144,7 +144,7 @@ public class CuratorDatabaseClient { } transaction.commit(); - nodes.forEach(node -> log.log(LogLevel.INFO, "Removed node " + node.hostname() + " in state " + node.state())); + nodes.forEach(node -> log.log(Level.INFO, "Removed node " + node.hostname() + " in state " + node.state())); } /** @@ -225,7 +225,7 @@ public class CuratorDatabaseClient { transaction.onCommitted(() -> { // schedule logging on commit of nodes which changed state for (Node node : nodes) { if (toState != node.state()) - log.log(LogLevel.INFO, agent + " moved " + node + " to " + toState + reason.map(s -> ": " + s).orElse("")); + log.log(Level.INFO, agent + " moved " + node + " to " + toState + reason.map(s -> ": " + s).orElse("")); } }); return writtenNodes; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index 8a31b271406..c67565db7a9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -57,7 +57,7 @@ public class InfraDeployerImpl implements InfraDeployer { try { deployment.activate(); } catch (RuntimeException e) { - logger.log(LogLevel.INFO, "Failed to activate " + application, e); + logger.log(Level.INFO, "Failed to activate " + application, e); if (propagateException) { throw e; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java index 0b439b81ef6..6a118f9d606 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java @@ -83,13 +83,13 @@ public class ApplicationSuspensionResource implements ApplicationSuspensionApi { try { orchestrator.suspend(applicationId); } catch (ApplicationIdNotFoundException e) { - log.log(LogLevel.INFO, "ApplicationId " + applicationIdString + " not found.", e); + log.log(Level.INFO, "ApplicationId " + applicationIdString + " not found.", e); throw new NotFoundException(e); } catch (ApplicationStateChangeDeniedException e) { - log.log(LogLevel.INFO, "Suspend for " + applicationIdString + " failed.", e); + log.log(Level.INFO, "Suspend for " + applicationIdString + " failed.", e); throw new WebApplicationException(Response.Status.CONFLICT); } catch (RuntimeException e) { - log.log(LogLevel.INFO, "Suspend for " + applicationIdString + " failed from unknown reasons", e); + log.log(Level.INFO, "Suspend for " + applicationIdString + " failed from unknown reasons", e); throw new InternalServerErrorException(e); } } @@ -100,13 +100,13 @@ public class ApplicationSuspensionResource implements ApplicationSuspensionApi { try { orchestrator.resume(applicationId); } catch (ApplicationIdNotFoundException e) { - log.log(LogLevel.INFO, "ApplicationId " + applicationIdString + " not found.", e); + log.log(Level.INFO, "ApplicationId " + applicationIdString + " not found.", e); throw new NotFoundException(e); } catch (ApplicationStateChangeDeniedException e) { - log.log(LogLevel.INFO, "Suspend for " + applicationIdString + " failed.", e); + log.log(Level.INFO, "Suspend for " + applicationIdString + " failed.", e); throw new WebApplicationException(Response.Status.CONFLICT); } catch (RuntimeException e) { - log.log(LogLevel.INFO, "Suspend for " + applicationIdString + " failed from unknown reasons", e); + log.log(Level.INFO, "Suspend for " + applicationIdString + " failed from unknown reasons", e); throw new InternalServerErrorException(e); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index d64b2342eaf..766218f8be5 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -102,7 +102,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi lockedRunnable(() -> { if (!superModelIsComplete) { superModelIsComplete = true; - logger.log(LogLevel.INFO, "All bootstrap tenant applications have been activated"); + logger.log(Level.INFO, "All bootstrap tenant applications have been activated"); maybeSetDuperModelAsComplete(); } }); @@ -171,7 +171,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi lockedRunnable(() -> { if (!infraApplicationsIsComplete) { infraApplicationsIsComplete = true; - logger.log(LogLevel.INFO, "All infrastructure applications have been activated"); + logger.log(Level.INFO, "All infrastructure applications have been activated"); maybeSetDuperModelAsComplete(); } }); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java index 6f2ccf70b2c..bfd8d47eb09 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java @@ -244,7 +244,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler { buckets.append(" "); buckets.append(bid.toString()); } - log.log(LogLevel.INFO, "Got EmptyBuckets: " + buckets); + log.log(Level.INFO, "Got EmptyBuckets: " + buckets); } @Override diff --git a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java index 0916c90bb2d..f3a2540b244 100644 --- a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -173,7 +173,7 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna public void run() { System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true"); String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; - log.log(LogLevel.INFO, "Starting ZooKeeper server with config file " + args[0] + + log.log(Level.INFO, "Starting ZooKeeper server with config file " + args[0] + ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); } -- cgit v1.2.3 From ee9e892838733da49d5ea2b74e456751c05c3f18 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 15:43:42 +0200 Subject: LogLevel.WARNING -> Level.WARNING --- .../instanceconfirmation/InstanceValidator.java | 10 +++++----- .../clustercontroller/core/FleetController.java | 10 +++++----- .../yahoo/vespa/clustercontroller/core/NodeInfo.java | 2 +- .../clustercontroller/core/NodeStateGatherer.java | 2 +- .../core/database/DatabaseHandler.java | 12 ++++++------ .../core/database/MasterDataGatherer.java | 6 +++--- .../core/database/ZooKeeperDatabase.java | 20 ++++++++++---------- .../vespa/clustercontroller/core/rpc/RpcServer.java | 4 ++-- .../clustercontroller/core/rpc/SlobrokClient.java | 2 +- .../core/status/statuspage/StatusPageServer.java | 4 ++-- .../clustercontroller/core/FleetControllerTest.java | 2 +- .../config/application/PropertiesProcessor.java | 2 +- .../main/java/com/yahoo/config/application/Xml.java | 2 +- .../provider/StaticConfigDefinitionRepo.java | 2 +- .../yahoo/config/model/builder/xml/XmlHelper.java | 2 +- .../application/validation/RankSetupValidator.java | 4 ++-- .../yahoo/vespa/model/builder/UserConfigBuilder.java | 4 ++-- .../model/builder/xml/dom/DomAdminV2Builder.java | 2 +- .../java/com/yahoo/vespa/model/utils/FileSender.java | 2 +- .../vespa/config/proxy/ConfigProxyRpcServer.java | 4 ++-- .../vespa/config/proxy/DelayedResponseHandler.java | 4 ++-- .../com/yahoo/vespa/config/proxy/MemoryCache.java | 2 +- .../vespa/config/proxy/RpcConfigSourceClient.java | 2 +- .../vespa/config/proxy/UpstreamConfigSubscriber.java | 2 +- .../filedistribution/CachedFilesMaintainer.java | 2 +- .../proxy/filedistribution/RequestTracker.java | 4 ++-- .../vespa/configserver/flags/http/FlagsHandler.java | 2 +- .../vespa/config/server/ApplicationRepository.java | 10 +++++----- .../vespa/config/server/ConfigServerBootstrap.java | 4 ++-- .../config/server/UserConfigDefinitionRepo.java | 2 +- .../CompressedApplicationInputStream.java | 4 ++-- .../server/application/FileDistributionStatus.java | 2 +- .../vespa/config/server/application/HttpProxy.java | 2 +- .../vespa/config/server/deploy/ZooKeeperClient.java | 2 +- .../server/filedistribution/FileDirectory.java | 6 +++--- .../config/server/filedistribution/FileServer.java | 2 +- .../yahoo/vespa/config/server/http/HttpHandler.java | 2 +- .../vespa/config/server/http/SimpleHttpFetcher.java | 6 +++--- .../config/server/modelfactory/ModelsBuilder.java | 4 ++-- .../config/server/rpc/DelayedConfigResponses.java | 2 +- .../vespa/config/server/rpc/GetConfigProcessor.java | 2 +- .../com/yahoo/vespa/config/server/rpc/RpcServer.java | 2 +- .../config/server/session/LocalSessionRepo.java | 4 ++-- .../server/session/LocalSessionStateWatcher.java | 4 ++-- .../config/server/session/RemoteSessionRepo.java | 2 +- .../server/session/RemoteSessionStateWatcher.java | 4 ++-- .../vespa/config/server/tenant/TenantRepository.java | 4 ++-- .../config/server/zookeeper/InitializedCounter.java | 6 +++--- .../server/deploy/DeployHandlerLoggerTest.java | 2 +- .../yahoo/container/jdisc/LoggingRequestHandler.java | 4 ++-- .../container/jdisc/ThreadedRequestHandler.java | 2 +- .../rendering/AsynchronousSectionedRenderer.java | 2 +- .../prelude/querytransform/StemmingSearcher.java | 2 +- .../yahoo/prelude/statistics/StatisticsSearcher.java | 2 +- .../search/grouping/UniqueGroupingSearcher.java | 2 +- .../java/com/yahoo/search/handler/SearchHandler.java | 2 +- .../yahoo/search/rendering/SyncDefaultRenderer.java | 2 +- .../tracing/LoggingTraceExporter.java | 2 +- .../certificate/EndpointCertificateManager.java | 4 ++-- .../controller/deployment/DeploymentTrigger.java | 4 ++-- .../hosted/controller/dns/NameServiceForwarder.java | 2 +- .../hosted/controller/dns/NameServiceQueue.java | 2 +- .../maintenance/ContactInformationMaintainer.java | 2 +- .../maintenance/DeploymentMetricsMaintainer.java | 2 +- .../hosted/controller/maintenance/JobRunner.java | 4 ++-- .../maintenance/OsVersionStatusUpdater.java | 2 +- .../maintenance/RotationStatusUpdater.java | 2 +- .../controller/maintenance/VersionStatusUpdater.java | 2 +- .../filter/ControllerAuthorizationFilter.java | 2 +- .../restapi/systemflags/SystemFlagsDeployer.java | 8 ++++---- .../hosted/controller/versions/VersionStatus.java | 2 +- .../yahoo/docproc/jdisc/DocumentProcessingTask.java | 2 +- .../messagebus/MessageBusVisitorSession.java | 4 ++-- .../messagebus/protocol/StoragePolicy.java | 2 +- .../messagebus/protocol/SubsetServicePolicy.java | 2 +- .../filedistribution/CompressedFileReference.java | 4 ++-- .../yahoo/vespa/filedistribution/FileDownloader.java | 2 +- .../yahoo/vespa/filedistribution/FileReceiver.java | 2 +- .../filedistribution/FileReferenceDownloader.java | 2 +- .../java/com/yahoo/messagebus/jdisc/MbusClient.java | 4 ++-- .../src/main/java/com/yahoo/logserver/Flusher.java | 2 +- .../handlers/logmetrics/LogMetricsHandler.java | 2 +- .../main/java/com/yahoo/messagebus/TraceNode.java | 10 +++++----- .../com/yahoo/messagebus/network/rpc/RPCTarget.java | 2 +- .../metricsproxy/service/ConfigSentinelClient.java | 4 ++-- .../metricsproxy/service/HttpMetricFetcher.java | 2 +- .../node/admin/nodeadmin/NodeAdminStateUpdater.java | 4 ++-- .../hosted/node/admin/nodeagent/NodeAgentImpl.java | 2 +- .../admin/task/util/process/ChildProcess2Impl.java | 2 +- .../admin/task/util/process/ProcessFactoryImpl.java | 2 +- .../maintenance/CapacityReportMaintainer.java | 2 +- .../provision/maintenance/LoadBalancerExpirer.java | 4 ++-- .../provision/maintenance/MaintenanceDeployment.java | 2 +- .../provision/provisioning/NodePrioritizer.java | 2 +- .../vespa/orchestrator/status/ZkApplicationLock.java | 2 +- .../vespa/orchestrator/status/ZkStatusService.java | 2 +- .../com/yahoo/vespa/service/duper/DuperModel.java | 2 +- .../vespa/service/executor/RunletExecutorImpl.java | 2 +- .../vespa/service/health/StateV1HealthClient.java | 2 +- .../service/slobrok/SlobrokMonitorManagerImpl.java | 2 +- .../main/java/com/yahoo/metrics/simple/Bucket.java | 2 +- .../java/com/yahoo/metrics/simple/UntypedMetric.java | 2 +- .../src/main/java/com/yahoo/statistics/Value.java | 2 +- .../client/AthenzIdentityProviderImpl.java | 4 ++-- .../com/yahoo/document/restapi/resource/RestApi.java | 2 +- .../com/yahoo/vespa/http/server/ClientFeederV3.java | 4 ++-- .../com/yahoo/vespa/http/server/FeedHandlerV3.java | 4 ++-- .../com/yahoo/vespa/http/server/FeedReplyReader.java | 2 +- .../main/java/com/yahoo/feedapi/SharedSender.java | 2 +- .../vespa/zookeeper/VespaZooKeeperServerImpl.java | 2 +- 110 files changed, 178 insertions(+), 178 deletions(-) diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index b101f0bce67..cfc63df4a92 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -77,7 +77,7 @@ public class InstanceValidator { VespaUniqueInstanceId csrProviderUniqueId = getVespaUniqueInstanceId(instanceConfirmation); if(! providerUniqueId.equals(csrProviderUniqueId)) { - log.log(LogLevel.WARNING, String.format("Instance %s has invalid provider unique ID in CSR (%s)", providerUniqueId, csrProviderUniqueId)); + log.log(Level.WARNING, String.format("Instance %s has invalid provider unique ID in CSR (%s)", providerUniqueId, csrProviderUniqueId)); return false; } @@ -107,7 +107,7 @@ public class InstanceValidator { try { return validateAttributes(confirmation, getVespaUniqueInstanceId(confirmation)); } catch (Exception e) { - log.log(LogLevel.WARNING, "Encountered exception while refreshing certificate for confirmation: " + confirmation, e); + log.log(Level.WARNING, "Encountered exception while refreshing certificate for confirmation: " + confirmation, e); return false; } } @@ -131,7 +131,7 @@ public class InstanceValidator { private boolean validateAttributes(InstanceConfirmation confirmation, VespaUniqueInstanceId vespaUniqueInstanceId) { if(vespaUniqueInstanceId == null) { - log.log(LogLevel.WARNING, "Unable to find unique instance ID in refresh request: " + confirmation.toString()); + log.log(Level.WARNING, "Unable to find unique instance ID in refresh request: " + confirmation.toString()); return false; } @@ -142,7 +142,7 @@ public class InstanceValidator { .findFirst() // Should be only one .orElse(null); if(node == null) { - log.log(LogLevel.WARNING, "Invalid InstanceConfirmation, No nodes matching uniqueId: " + vespaUniqueInstanceId); + log.log(Level.WARNING, "Invalid InstanceConfirmation, No nodes matching uniqueId: " + vespaUniqueInstanceId); return false; } @@ -162,7 +162,7 @@ public class InstanceValidator { // Validate that ipaddresses in request are valid for node if(! nodeIpAddresses.containsAll(ips)) { - log.log(LogLevel.WARNING, "Invalid InstanceConfirmation, wrong ip in : " + vespaUniqueInstanceId); + log.log(Level.WARNING, "Invalid InstanceConfirmation, wrong ip in : " + vespaUniqueInstanceId); return false; } return true; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index bfa202432b6..b9c839c3b87 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -510,9 +510,9 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd try{ rpcServer.setSlobrokConnectionSpecs(options.slobrokConnectionSpecs, options.rpcPort); } catch (ListenFailedException e) { - log.log(LogLevel.WARNING, "Failed to bind RPC server to port " + options.rpcPort +". This may be natural if cluster has altered the services running on this node: " + e.getMessage()); + log.log(Level.WARNING, "Failed to bind RPC server to port " + options.rpcPort +". This may be natural if cluster has altered the services running on this node: " + e.getMessage()); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to initialize RPC server socket: " + e.getMessage()); + log.log(Level.WARNING, "Failed to initialize RPC server socket: " + e.getMessage()); } } @@ -520,7 +520,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd try{ statusPageServer.setPort(options.httpPort); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to initialize status server socket. This may be natural if cluster has altered the services running on this node: " + e.getMessage()); + log.log(Level.WARNING, "Failed to initialize status server socket. This may be natural if cluster has altered the services running on this node: " + e.getMessage()); } } @@ -633,7 +633,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } catch (InterruptedException e) { throw (InterruptedException) new InterruptedException("Interrupted").initCause(e); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to watch master election: " + e.toString()); + log.log(Level.WARNING, "Failed to watch master election: " + e.toString()); } return false; } @@ -811,7 +811,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd taskCompletionQueue.remove(); } else if (taskCompletion.getDeadlineTimePointMs() <= now) { var details = buildNodesNotYetConvergedMessage(taskCompletion.getMinimumVersion()); - log.log(LogLevel.WARNING, () -> String.format("Deferred task of type '%s' has exceeded wait deadline; completing with failure (details: %s)", + log.log(Level.WARNING, () -> String.format("Deferred task of type '%s' has exceeded wait deadline; completing with failure (details: %s)", taskCompletion.getTask().getClass().getName(), details)); taskCompletion.getTask().handleFailure(RemoteClusterControllerTask.Failure.of( RemoteClusterControllerTask.FailureCondition.DEADLINE_EXCEEDED, details)); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java index 12a6d72d400..652c3039430 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java @@ -392,7 +392,7 @@ abstract public class NodeInfo implements Comparable { return true; } } - log.log(LogLevel.WARNING, "Node " + toString() + " does not support " + methodName + " which it should."); + log.log(Level.WARNING, "Node " + toString() + " does not support " + methodName + " which it should."); return false; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java index 7e7092eb67e..87a572ce4a0 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java @@ -135,7 +135,7 @@ public class NodeStateGatherer { listener.handleNewNodeState(info, state.clone()); info.setReportedState(state, currentTime); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to process get node state response", e); + log.log(Level.WARNING, "Failed to process get node state response", e); info.setReportedState(new NodeState(info.getNode().getType(), State.DOWN), currentTime); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java index f648f08a3ea..bb0692c2e24 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java @@ -181,12 +181,12 @@ public class DatabaseHandler { } catch (InterruptedException e) { throw (InterruptedException) new InterruptedException("Interrupted").initCause(e); } catch (KeeperException.ConnectionLossException e) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to connect to ZooKeeper at " + zooKeeperAddress + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to connect to ZooKeeper at " + zooKeeperAddress + " with session timeout " + zooKeeperSessionTimeout + ": " + e.getMessage()); } catch (Exception e) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to connect to ZooKeeper at " + zooKeeperAddress + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to connect to ZooKeeper at " + zooKeeperAddress + " with session timeout " + zooKeeperSessionTimeout + ": " + sw); } log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Done setting up new ZooKeeper session at " + zooKeeperAddress); @@ -239,7 +239,7 @@ public class DatabaseHandler { didWork |= performZooKeeperWrites(); } } catch (CasWriteFailed e) { - log.log(LogLevel.WARNING, String.format("CaS write to ZooKeeper failed, another controller " + + log.log(Level.WARNING, String.format("CaS write to ZooKeeper failed, another controller " + "has likely taken over ownership: %s", e.getMessage())); // Clear DB and master election state. This shall trigger a full re-fetch of all // version and election-related metadata. @@ -265,7 +265,7 @@ public class DatabaseHandler { currentlyStored.masterVote = pendingStore.masterVote; pendingStore.masterVote = null; } else { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to store master vote"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to store master vote"); return true; } } @@ -355,7 +355,7 @@ public class DatabaseHandler { Integer version = currentlyStored.lastSystemStateVersion; if (version == null) { if (usingZooKeeper()) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to retrieve latest system state version from ZooKeeper. Returning version 0."); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to retrieve latest system state version from ZooKeeper. Returning version 0."); } return 0; // FIXME "fail-oblivious" is not a good error handling mode for such a critical component! } @@ -467,7 +467,7 @@ public class DatabaseHandler { Map startTimestamps = currentlyStored.startTimestamps; if (startTimestamps == null) { if (usingZooKeeper()) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to retrieve start timestamps from ZooKeeper. Cluster state will be bloated with timestamps until we get them set."); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to retrieve start timestamps from ZooKeeper. Cluster state will be bloated with timestamps until we get them set."); } startTimestamps = new TreeMap<>(); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java index 1c50afa4310..75dfb90d059 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java @@ -63,7 +63,7 @@ public class MasterDataGatherer { session.getData(zooKeeperRoot + "indexes/" + index, this, nodeListener, null); break; case NodeCreated: // How can this happen? Can one leave watches on non-existing nodes? - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeCreated"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeCreated"); break; case NodeDeleted: // We get this event when fleetcontrollers shut down and node in dir disappears. But it should also trigger a NodeChildrenChanged event, so @@ -115,7 +115,7 @@ public class MasterDataGatherer { log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Node at " + path + " removed, got no other option than counting it as down."); } else { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failure code " + code + + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failure code " + code + " when listening to node at " + path + ", will assume it's down."); } if (nextMasterData.containsKey(index)) { @@ -133,7 +133,7 @@ public class MasterDataGatherer { nextMasterData.put(index, value); } } else { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + " which is not alive according to current state. Ignoring it"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + " which is not alive according to current state. Ignoring it"); } } for(Integer vote : nextMasterData.values()) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java index 7e878c80fdb..bba417b1333 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java @@ -52,7 +52,7 @@ public class ZooKeeperDatabase extends Database { public void process(WatchedEvent watchedEvent) { // Shouldn't get events after we expire, but just be sure we stop them here. if (state != null && state.equals(Event.KeeperState.Expired)) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got event from ZooKeeper session after it expired"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got event from ZooKeeper session after it expired"); return; } Event.KeeperState newState = watchedEvent.getState(); @@ -75,20 +75,20 @@ public class ZooKeeperDatabase extends Database { } switch (watchedEvent.getType()) { case NodeChildrenChanged: // Fleetcontrollers have either connected or disconnected to ZooKeeper - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeChildrenChanged"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeChildrenChanged"); break; case NodeDataChanged: // A fleetcontroller have changed what node it is voting for - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeDataChanged"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeDataChanged"); break; case NodeCreated: // How can this happen? Can one leave watches on non-existing nodes? - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeCreated"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeCreated"); break; case NodeDeleted: // We're not watching any nodes for whether they are deleted or not. - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeDeleted"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeDeleted"); break; case None: if (state != null && state.equals(watchedEvent.getState())) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got None type event that didn't even alter session state. What does that indicate?"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got None type event that didn't even alter session state. What does that indicate?"); } } state = watchedEvent.getState(); @@ -166,7 +166,7 @@ public class ZooKeeperDatabase extends Database { + Long.toHexString(session.getSessionId())); session.close(); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got interrupt exception while closing session: " + e); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got interrupt exception while closing session: " + e); } } @@ -178,7 +178,7 @@ public class ZooKeeperDatabase extends Database { if (sessionOpen && reportErrors) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); - log.log(LogLevel.WARNING, String.format("Fleetcontroller %s: %s. Exception: %s\n%s", + log.log(Level.WARNING, String.format("Fleetcontroller %s: %s. Exception: %s\n%s", nodeIndex, message, e.getMessage(), sw.toString())); } } @@ -277,7 +277,7 @@ public class ZooKeeperDatabase extends Database { NodeState nodeState = NodeState.deserialize(node.getType(), token.substring(colon + 1)); wanted.put(node, nodeState); } catch (Exception e) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Ignoring invalid wantedstate line in zookeeper '" + token + "'."); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Ignoring invalid wantedstate line in zookeeper '" + token + "'."); } } } @@ -329,7 +329,7 @@ public class ZooKeeperDatabase extends Database { Long timestamp = Long.valueOf(token.substring(colon + 1)); wanted.put(n, timestamp); } catch (Exception e) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Ignoring invalid starttimestamp line in zookeeper '" + token + "'."); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Ignoring invalid starttimestamp line in zookeeper '" + token + "'."); } } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index c12f906fe09..fb23e54b82a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -197,13 +197,13 @@ public class RpcServer { if (!e.getMessage().equals(lastConnectError) || time - lastConnectErrorTime > 60 * 1000) { lastConnectError = e.getMessage(); lastConnectErrorTime = time; - log.log(LogLevel.WARNING, "Failed to bind RPC server to port " + port +": " + e.getMessage()); + log.log(Level.WARNING, "Failed to bind RPC server to port " + port +": " + e.getMessage()); } } catch (Exception e) { if (!e.getMessage().equals(lastConnectError) || time - lastConnectErrorTime > 60 * 1000) { lastConnectError = e.getMessage(); lastConnectErrorTime = time; - log.log(LogLevel.WARNING, "Failed to initailize RPC server socket: " + e.getMessage()); + log.log(Level.WARNING, "Failed to initailize RPC server socket: " + e.getMessage()); } } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index 643a6a94c50..3ef26b2a19f 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -142,7 +142,7 @@ public class SlobrokClient implements NodeLookup { cluster.setSlobrokGenerationCount(mirrorVersion); for (NodeInfo nodeInfo : cluster.getNodeInfo()) { if (slobrokNodes.containsKey(nodeInfo.getNode()) && nodeInfo.isRpcAddressOutdated()) { - log.log(LogLevel.WARNING, "Node " + nodeInfo + log.log(Level.WARNING, "Node " + nodeInfo + " was tagged NOT in slobrok even though it is. It was in the following lists:" + (newNodes.contains(nodeInfo.getNode()) ? " newNodes" : "") + (missingNodeInfos.contains(nodeInfo) ? " missingNodes" : "") diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index 8b3bec1b2cf..55241d8310d 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -238,7 +238,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? LogLevel.DEBUG : LogLevel.INFO, "Failed to process HTTP request : " + e.getMessage()); } catch (Exception e) { - log.log(LogLevel.WARNING, "Caught exception in HTTP server thread: " + log.log(Level.WARNING, "Caught exception in HTTP server thread: " + e.getClass().getName() + ": " + e.getMessage()); } finally { if (output != null) try { @@ -392,7 +392,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { if (!e.getMessage().equals(lastConnectError) || time - lastConnectErrorTime > 60 * 1000) { lastConnectError = e.getMessage(); lastConnectErrorTime = time; - log.log(LogLevel.WARNING, "Failed to initialize HTTP status server server socket: " + e.getMessage()); + log.log(Level.WARNING, "Failed to initialize HTTP status server server socket: " + e.getMessage()); } } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index fa2eb80506e..5c910983bc9 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -211,7 +211,7 @@ public abstract class FleetControllerTest implements Waiter { if (fleetController == null) { fleetController = createFleetController(usingFakeTimer, options, true, null); } else { - log.log(LogLevel.WARNING, "already started fleetcontroller, not starting another"); + log.log(Level.WARNING, "already started fleetcontroller, not starting another"); } } diff --git a/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java index 2966030db40..1c5ea8510cb 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java @@ -42,7 +42,7 @@ class PropertiesProcessor implements PreProcessor { //System.out.println("Found " + node.getNodeName() + ", " + node.getTextContent()); final String propertyName = node.getNodeName(); if (properties.containsKey(propertyName)) { - log.log(LogLevel.WARNING, "Duplicate definition for property '" + propertyName + "' detected"); + log.log(Level.WARNING, "Duplicate definition for property '" + propertyName + "' detected"); } properties.put(propertyName, node.getTextContent()); } diff --git a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java index afc6634b4cd..c48a41083c7 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java @@ -61,7 +61,7 @@ public class Xml { try { return factory.newDocumentBuilder(); } catch (ParserConfigurationException e) { - log.log(LogLevel.WARNING, "No XML parser available - " + e); + log.log(Level.WARNING, "No XML parser available - " + e); return null; } } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java index 8faef6b2ce6..c930adfb115 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java @@ -47,7 +47,7 @@ public class StaticConfigDefinitionRepo implements ConfigDefinitionRepo { ConfigDefinitionKey key = ConfigUtils.createConfigDefinitionKeyFromDefFile(def); addConfigDefinition(key, def); } catch (IOException e) { - log.log(LogLevel.WARNING, "Exception adding config definition " + def, e); + log.log(Level.WARNING, "Exception adding config definition " + def, e); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java index 93f308a2c2e..a392a778709 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java @@ -124,7 +124,7 @@ public final class XmlHelper { log.log(Level.FINE, "XML parser now operational!"); return docBuilder; } catch (ParserConfigurationException e) { - log.log(LogLevel.WARNING, "No XML parser available - " + e); + log.log(Level.WARNING, "No XML parser available - " + e); return null; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 8437df43df9..96dc312abd7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -154,7 +154,7 @@ public class RankSetupValidator extends Validator { private void validateWarn(Exception e, DeployLogger deployLogger) { String msg = "Unable to execute '"+ binaryName + "', validation of rank expressions will only take place when you start Vespa: " + Exceptions.toMessageString(e); - deployLogger.log(LogLevel.WARNING, msg); + deployLogger.log(Level.WARNING, msg); } private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger deployLogger) { @@ -171,7 +171,7 @@ public class RankSetupValidator extends Validator { } } if (ignoreValidationErrors) { - deployLogger.log(LogLevel.WARNING, errMsg.append("(Continuing since ignoreValidationErrors flag is set.)").toString()); + deployLogger.log(Level.WARNING, errMsg.append("(Continuing since ignoreValidationErrors flag is set.)").toString()); } else { throw new IllegalArgumentException(errMsg.toString()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java index 00549cc3a99..be85861ddb6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java @@ -40,13 +40,13 @@ public class UserConfigBuilder { Optional def = configDefinitionStore.getConfigDefinition(key); if ( ! def.isPresent()) { // TODO: Fail instead of warn - logger.log(LogLevel.WARNING, "Unable to find config definition '" + key.asFileName() + + logger.log(Level.WARNING, "Unable to find config definition '" + key.asFileName() + "'. Please ensure that the name is spelled correctly, and that the def file is included in a bundle."); } ConfigPayloadBuilder payloadBuilder = new DomConfigPayloadBuilder(def.orElse(null)).build(element); ConfigPayloadBuilder old = builderMap.get(key); if (old != null) { - logger.log(LogLevel.WARNING, "Multiple overrides for " + key + " found. Applying in the order they are discovered"); + logger.log(Level.WARNING, "Multiple overrides for " + key + " found. Applying in the order they are discovered"); old.override(payloadBuilder); } else { builderMap.put(key, payloadBuilder); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 1afb3abe90f..706311dcdbf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -53,7 +53,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { addLogForwarders(adminElement.child("logforwarding"), admin); if (adminElement.child("filedistribution") != null) { - deployState.getDeployLogger().log(LogLevel.WARNING, "'filedistribution' element is deprecated and ignored"); + deployState.getDeployLogger().log(Level.WARNING, "'filedistribution' element is deprecated and ignored"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index 901d81927a5..6e6a4d5ac8f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -84,7 +84,7 @@ public class FileSender implements Serializable { ConfigDefinition configDefinition = builder.getConfigDefinition(); if (configDefinition == null) { // TODO: throw new IllegalArgumentException("Not able to find config definition for " + builder); - logger.log(LogLevel.WARNING, "Not able to find config definition for " + key + ". Will not send files for this config"); + logger.log(Level.WARNING, "Not able to find config definition for " + key + ". Will not send files for this config"); return; } // Inspect fields at this level diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index 1399c3677af..a19d5809660 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -253,7 +253,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer log.log(Level.FINEST, () -> String.format("Executing RPC request %s.", requestLogId(request))); handler.run(); } catch (Exception e) { - log.log(LogLevel.WARNING, + log.log(Level.WARNING, String.format("Exception thrown during execution of RPC request %s: %s", requestLogId(request), e.getMessage()), e); } }); @@ -273,7 +273,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer log.log(Level.FINE, () ->"getConfig: " + request.getShortDescription() + ",configmd5=" + request.getRequestConfigMd5()); if (!request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. - log.log(LogLevel.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); + log.log(Level.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); returnErrorResponse(request, request.errorCode(), "Parameters for request " + request.getShortDescription() + " did not validate: " + request.errorMessage()); return; } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java index 42bb2d21b55..317606ac3ef 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java @@ -50,14 +50,14 @@ public class DelayedResponseHandler implements Runnable { rpcServer.returnOkResponse(request, config); i.incrementAndGet(); } else { - log.log(LogLevel.WARNING, "Timed out (timeout " + request.getTimeout() + ") getting config " + + log.log(Level.WARNING, "Timed out (timeout " + request.getTimeout() + ") getting config " + request.getConfigKey() + ", will retry"); } } log.log(Level.FINEST, () -> "Finished running DelayedResponseHandler. " + i.get() + " delayed responses sent in " + (System.currentTimeMillis() - start) + " ms"); } catch (Exception e) { // To avoid thread throwing exception and executor never running this again - log.log(LogLevel.WARNING, "Got exception in DelayedResponseHandler: " + Exceptions.toMessageString(e)); + log.log(Level.WARNING, "Got exception in DelayedResponseHandler: " + Exceptions.toMessageString(e)); } catch (Throwable e) { com.yahoo.protect.Process.logAndDie("Got error in DelayedResponseHandler, exiting: " + Exceptions.toMessageString(e)); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java index c7a79cc8d14..aae76c4f980 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java @@ -131,7 +131,7 @@ public class MemoryCache { writer.write("\n"); writer.close(); } catch (IOException e) { - log.log(LogLevel.WARNING, "Could not write to file '" + filename + "'"); + log.log(Level.WARNING, "Could not write to file '" + filename + "'"); } finally { if (writer != null) { try { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index 25961caf85b..12fd95b8503 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -84,7 +84,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { */ private void checkConfigSources() { if (configSourceSet == null || configSourceSet.getSources() == null || configSourceSet.getSources().size() == 0) { - log.log(LogLevel.WARNING, "No config sources defined, could not check connection"); + log.log(Level.WARNING, "No config sources defined, could not check connection"); } else { Request req = new Request("ping"); for (String configSource : configSourceSet.getSources()) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java index 14d93ee046e..a24407588be 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java @@ -56,7 +56,7 @@ public class UpstreamConfigSubscriber implements Subscriber { try { updateWithNewConfig(handle); } catch (Exception e) { // To avoid thread throwing exception and loop never running this again - log.log(LogLevel.WARNING, "Got exception: " + Exceptions.toMessageString(e)); + log.log(Level.WARNING, "Got exception: " + Exceptions.toMessageString(e)); } catch (Throwable e) { com.yahoo.protect.Process.logAndDie("Got error, exiting: " + Exceptions.toMessageString(e)); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java index ffe7f7bec46..247ef8a4dbf 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java @@ -74,7 +74,7 @@ class CachedFilesMaintainer implements Runnable { filesToDelete.forEach(fileReference -> { File file = new File(directory, fileReference); if (!IOUtils.recursiveDeleteDir(file)) - log.log(LogLevel.WARNING, "Could not delete " + file.getAbsolutePath()); + log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath()); }); } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java index 6b0be5bca16..206c0359850 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java @@ -21,10 +21,10 @@ class RequestTracker { void trackRequest(File file) { String absolutePath = file.getAbsolutePath(); if ( ! file.exists()) - log.log(LogLevel.WARNING, "Could not find file '" + absolutePath + "'"); + log.log(Level.WARNING, "Could not find file '" + absolutePath + "'"); if ( ! file.setLastModified(Instant.now().toEpochMilli())) - log.log(LogLevel.WARNING, "Could not set last modified timestamp for '" + absolutePath + "'"); + log.log(Level.WARNING, "Could not set last modified timestamp for '" + absolutePath + "'"); } } diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java index 64cae0f31a0..c034ea79447 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java @@ -48,7 +48,7 @@ public class FlagsHandler extends LoggingRequestHandler { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(LogLevel.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 583d560994e..f6846a7448e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -447,7 +447,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .map(FileReference::value) .collect(Collectors.toSet())); } catch (Exception e) { - log.log(LogLevel.WARNING, "Getting file references in use for '" + application + "' failed", e); + log.log(Level.WARNING, "Getting file references in use for '" + application + "' failed", e); } } log.log(Level.FINE, "File references in use : " + fileReferencesInUse); @@ -470,7 +470,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye fileReferencesToDelete.forEach(fileReference -> { File file = new File(fileReferencesPath, fileReference); if ( ! IOUtils.recursiveDeleteDir(file)) - log.log(LogLevel.WARNING, "Could not delete " + file.getAbsolutePath()); + log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath()); }); } return fileReferencesToDelete; @@ -498,10 +498,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (session == null) throw new NotFoundException("Remote session " + sessionId + " not found"); return session.ensureApplicationLoaded().getForVersionOrLatest(version, clock.instant()); } catch (NotFoundException e) { - log.log(LogLevel.WARNING, "Failed getting application for '" + applicationId + "': " + e.getMessage()); + log.log(Level.WARNING, "Failed getting application for '" + applicationId + "': " + e.getMessage()); throw e; } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed getting application for '" + applicationId + "'", e); + log.log(Level.WARNING, "Failed getting application for '" + applicationId + "'", e); throw e; } } @@ -785,7 +785,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private void cleanupTempDirectory(File tempDir) { logger.log(Level.FINE, "Deleting tmp dir '" + tempDir + "'"); if (!IOUtils.recursiveDeleteDir(tempDir)) { - logger.log(LogLevel.WARNING, "Not able to delete tmp dir '" + tempDir + "'"); + logger.log(Level.WARNING, "Not able to delete tmp dir '" + tempDir + "'"); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 35cbd0c4303..36767e6fc5f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -127,7 +127,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable try { thread.join(); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "Error joining server thread on shutdown: " + e.getMessage()); + log.log(Level.WARNING, "Error joining server thread on shutdown: " + e.getMessage()); } }); } @@ -246,7 +246,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable log.log(Level.INFO, "Redeploying " + app + " failed with transient error, will retry after bootstrap: " + Exceptions.toMessageString(e)); } else { - log.log(LogLevel.WARNING, "Redeploying " + app + " failed, will retry", e); + log.log(Level.WARNING, "Redeploying " + app + " failed, will retry", e); failedDeployments.add(app); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java index 2a632a75330..15a9b8f74d0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java @@ -32,7 +32,7 @@ public class UserConfigDefinitionRepo implements ConfigDefinitionRepo { defs.put(dKey, new ConfigDefinition(dKey.getName(), Splitter.on("\n").splitToList(payload).toArray(new String[0]))); } } else { - log.log(LogLevel.WARNING, "Path " + appPath + " does not exist, not able to load add user config definitions"); + log.log(Level.WARNING, "Path " + appPath + " does not exist, not able to load add user config definitions"); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index 2cab0ea5e20..eeb4d31fe12 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -96,7 +96,7 @@ public class CompressedApplicationInputStream implements AutoCloseable { log.log(Level.FINE, "Creating dir: " + outFile.getAbsolutePath()); boolean res = outFile.mkdirs(); if (!res) { - log.log(LogLevel.WARNING, "Could not create dir " + entry.getName()); + log.log(Level.WARNING, "Could not create dir " + entry.getName()); } } } else { @@ -113,7 +113,7 @@ public class CompressedApplicationInputStream implements AutoCloseable { entries++; } if (entries == 0) { - log.log(LogLevel.WARNING, "Not able to read any entries from " + application.getName()); + log.log(Level.WARNING, "Not able to read any entries from " + application.getName()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java index f00acacef03..092db354ffd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java @@ -56,7 +56,7 @@ public class FileDistributionStatus extends AbstractComponent { try { hostStatuses.add(future.get()); } catch (InterruptedException | ExecutionException e) { - log.log(LogLevel.WARNING, "Failed getting file distribution status", e); + log.log(Level.WARNING, "Failed getting file distribution status", e); } }); return createStatusForAllHosts(hostStatuses); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java index 88b13662b7f..8c53a9bc9d5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java @@ -58,7 +58,7 @@ public class HttpProxy { try { url = new URL(urlString); } catch (MalformedURLException e) { - logger.log(LogLevel.WARNING, "Badly formed url: " + urlString, e); + logger.log(Level.WARNING, "Badly formed url: " + urlString, e); return HttpErrorResponse.internalServerError("Failed to construct URL for backend"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 825a0b4a631..989fa10c202 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -298,7 +298,7 @@ public class ZooKeeperClient { configCurator.deleteRecurse(getZooKeeperAppPath(null).append(subPath).getAbsolute()); } } catch (Exception e) { - logger.log(LogLevel.WARNING, "Could not clean up in zookeeper"); + logger.log(Level.WARNING, "Could not clean up in zookeeper"); //Might be called in an exception handler before re-throw, so do not throw here. } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index c4279414a9b..cfa648ce351 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -32,7 +32,7 @@ public class FileDirectory { try { ensureRootExist(); } catch (IllegalArgumentException e) { - log.log(LogLevel.WARNING, "Failed creating directory in constructor, will retry on demand : " + e.toString()); + log.log(Level.WARNING, "Failed creating directory in constructor, will retry on demand : " + e.toString()); } } @@ -85,7 +85,7 @@ public class FileDirectory { log.log(Level.FINE, "Calculating hash for '" + path + "'"); return hash(path.toFile(), hasher); } catch (IOException e) { - log.log(LogLevel.WARNING, "Failed getting hash from '" + path + "'"); + log.log(Level.WARNING, "Failed getting hash from '" + path + "'"); return 0; } }).mapToLong(Number::longValue).sum(); @@ -153,7 +153,7 @@ public class FileDirectory { if (!destinationDir.exists()) { log.log(Level.FINE, "Moving from " + tempDestinationDir + " to " + destinationDir.getAbsolutePath()); if ( ! tempDestinationDir.toFile().renameTo(destinationDir)) { - log.log(LogLevel.WARNING, "Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'."); + log.log(Level.WARNING, "Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'."); } } else { IOUtils.copyDirectory(tempDestinationDir.toFile(), destinationDir, -1); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 6cde8cca18f..805ee2bef95 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -133,7 +133,7 @@ public class FileServer { target.receive(fileData, new ReplayStatus(success ? 0 : 1, success ? "OK" : errorDescription)); log.log(Level.FINE, "Done serving file reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed serving file reference '" + reference.value() + "': " + Exceptions.toMessageString(e)); + log.log(Level.WARNING, "Failed serving file reference '" + reference.value() + "': " + Exceptions.toMessageString(e)); } finally { fileData.close(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index e1a5845c4a1..e92eb52028b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -71,7 +71,7 @@ public class HttpHandler extends LoggingRequestHandler { } catch (LoadBalancerServiceException e) { return HttpErrorResponse.loadBalancerNotReady(getMessage(e, request)); } catch (Exception e) { - log.log(LogLevel.WARNING, "Unexpected exception handling a config server request", e); + log.log(Level.WARNING, "Unexpected exception handling a config server request", e); return HttpErrorResponse.internalServerError(getMessage(e, request)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java index 28dfa30e7ac..9c77442dcc8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java @@ -42,15 +42,15 @@ public class SimpleHttpFetcher implements HttpFetcher { } } catch (ConnectTimeoutException | SocketTimeoutException e) { String message = "Timed out after " + params.readTimeoutMs + " ms reading response from " + url; - logger.log(LogLevel.WARNING, message, e); + logger.log(Level.WARNING, message, e); throw new RequestTimeoutException(message); } catch (IOException e) { String message = "Failed to get response from " + url; - logger.log(LogLevel.WARNING, message, e); + logger.log(Level.WARNING, message, e); throw new InternalServerException(message); } catch (URISyntaxException e) { String message = "Invalid URL: " + e.getMessage(); - logger.log(LogLevel.WARNING, message, e); + logger.log(Level.WARNING, message, e); throw new InternalServerException(message, e); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 1aaa090d145..ebb399076ad 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -121,10 +121,10 @@ public abstract class ModelsBuilder { log.log(Level.INFO, applicationId + ": Skipping major version " + majorVersion, e); } else { if (e instanceof NullPointerException || e instanceof NoSuchElementException | e instanceof UncheckedTimeoutException) { - log.log(LogLevel.WARNING, "Unexpected error when building model ", e); + log.log(Level.WARNING, "Unexpected error when building model ", e); throw new InternalServerException(applicationId + ": Error loading model", e); } else { - log.log(LogLevel.WARNING, "Input error when building model ", e); + log.log(Level.WARNING, "Input error when building model ", e); throw new IllegalArgumentException(applicationId + ": Error loading model", e); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java index 3c18d5f11d2..cb7e5e4bf13 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java @@ -185,7 +185,7 @@ public class DelayedConfigResponses { response.schedule(Math.max(0, request.getTimeout())); metricDelayedResponses(context.applicationId(), delayedResponsesQueue.size()); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, context.logPre()+"Interrupted when putting on delayed requests queue."); + log.log(Level.WARNING, context.logPre()+"Interrupted when putting on delayed requests queue."); } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index f2099388c7c..d55235d4602 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -71,7 +71,7 @@ class GetConfigProcessor implements Runnable { //Request has already been detached if ( ! request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. - log.log(LogLevel.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); + log.log(Level.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); respond(request); return null; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 977e1f8cc28..22528e31dc0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -398,7 +398,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { String msg = TenantRepository.logPre(tenant) + "Unable to find request handler for tenant. Requested from host '" + request.getClientHostName() + "'"; metrics.incUnknownHostRequests(); trace.trace(TRACELEVEL, msg); - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); return null; } RequestHandler handler = requestHandler.get(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java index bf76cf67cf1..97c4530e447 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -71,7 +71,7 @@ public class LocalSessionRepo extends SessionRepo { try { addSession(loader.loadSession(Long.parseLong(session.getName()))); } catch (IllegalArgumentException e) { - log.log(LogLevel.WARNING, "Could not load session '" + + log.log(Level.WARNING, "Could not load session '" + session.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); } } @@ -88,7 +88,7 @@ public class LocalSessionRepo extends SessionRepo { } // Make sure to catch here, to avoid executor just dying in case of issues ... } catch (Throwable e) { - log.log(LogLevel.WARNING, "Error when purging old sessions ", e); + log.log(Level.WARNING, "Error when purging old sessions ", e); } log.log(Level.FINE, "Done purging old sessions"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java index 5a972086d82..268a755b266 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java @@ -54,7 +54,7 @@ public class LocalSessionStateWatcher { try { fileCache.close(); } catch (Exception e) { - log.log(LogLevel.WARNING, "Exception when closing watcher", e); + log.log(Level.WARNING, "Exception when closing watcher", e); } } @@ -66,7 +66,7 @@ public class LocalSessionStateWatcher { sessionChanged(Session.Status.parse(Utf8.toString(node.getData()))); } } catch (Exception e) { - log.log(LogLevel.WARNING, session.logPre() + "Error handling session changed for session " + getSessionId(), e); + log.log(Level.WARNING, session.logPre() + "Error handling session changed for session " + getSessionId(), e); } }); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index d66525326a2..6dade946e5d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -178,7 +178,7 @@ public class RemoteSessionRepo extends SessionRepo { directoryCache.close(); } } catch (Exception e) { - log.log(LogLevel.WARNING, "Exception when closing path cache", e); + log.log(Level.WARNING, "Exception when closing path cache", e); } finally { checkForRemovedSessions(new ArrayList<>()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java index 549ad2a2d24..eb4047e5fea 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java @@ -66,7 +66,7 @@ public class RemoteSessionStateWatcher { try { fileCache.close(); } catch (Exception e) { - log.log(LogLevel.WARNING, "Exception when closing watcher", e); + log.log(Level.WARNING, "Exception when closing watcher", e); } } @@ -78,7 +78,7 @@ public class RemoteSessionStateWatcher { sessionChanged(Session.Status.parse(Utf8.toString(node.getData()))); } } catch (Exception e) { - log.log(LogLevel.WARNING, session.logPre() + "Error handling session changed for session " + getSessionId(), e); + log.log(Level.WARNING, session.logPre() + "Error handling session changed for session " + getSessionId(), e); metrics.incSessionChangeErrors(); } }); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 49bfc20233e..04deed40c54 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -175,10 +175,10 @@ public class TenantRepository { try { f.getValue().get(); } catch (ExecutionException e) { - log.log(LogLevel.WARNING, "Failed to create tenant " + tenantName, e); + log.log(Level.WARNING, "Failed to create tenant " + tenantName, e); failed.add(tenantName); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "Interrupted while creating tenant '" + tenantName + "'", e); + log.log(Level.WARNING, "Interrupted while creating tenant '" + tenantName + "'", e); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java index 6bf645cc02c..389649f4680 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java @@ -45,7 +45,7 @@ public class InitializedCounter { try { return configCurator.exists(appsPath); } catch (Exception e) { - log.log(LogLevel.WARNING, e.getMessage()); + log.log(Level.WARNING, e.getMessage()); return false; } } @@ -64,7 +64,7 @@ public class InitializedCounter { newestGeneration = Collections.max(getDeployedApplicationGenerations(configCurator, appsPath)); } } catch (Exception e) { - log.log(LogLevel.WARNING, "Could not get newest application generation from Zookeeper"); + log.log(Level.WARNING, "Could not get newest application generation from Zookeeper"); } return newestGeneration; } @@ -79,7 +79,7 @@ public class InitializedCounter { } } } catch (RuntimeException e) { - log.log(LogLevel.WARNING, "Could not get application generations from Zookeeper"); + log.log(Level.WARNING, "Could not get application generations from Zookeeper"); } return generations; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java index a52788031c7..da450dfbedb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java @@ -45,6 +45,6 @@ public class DeployHandlerLoggerTest { logger.log(Level.FINE, "foobar"); logger.log(Level.FINEST, "foobar"); logger.log(LogLevel.FINE, "baz"); - logger.log(LogLevel.WARNING, "baz"); + logger.log(Level.WARNING, "baz"); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index 6878b6df09d..d7b846cef5c 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -166,7 +166,7 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { b.append("Timeout: ").append(timeoutInterval).append(" ms. "); b.append("Request string: ").append(req); - log.log(LogLevel.WARNING, "Slow execution. " + b); + log.log(Level.WARNING, "Slow execution. " + b); } private static class NullResponse extends ExtendedResponse { @@ -308,7 +308,7 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { logEntry.setHttpMethod(AccessLogUtil.getHttpMethod(httpRequest)); logEntry.setHttpVersion(AccessLogUtil.getHttpVersion(httpRequest)); } catch (Exception e) { - log.log(LogLevel.WARNING, "Could not populate the access log [" + fullRequest + "]", e); + log.log(Level.WARNING, "Could not populate the access log [" + fullRequest + "]", e); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index a34ceb7f613..34ba7676680 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -147,7 +147,7 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { currentFailureIntervalStartedMillis = 0L; numRejectedRequests = 0; } - log.log(LogLevel.WARNING, "Rejected " + numRejectedRequestsSnapshot + " requests on cause of no available worker threads."); + log.log(Level.WARNING, "Rejected " + numRejectedRequestsSnapshot + " requests on cause of no available worker threads."); } private void incrementRejectedRequests() { diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java index 3a1eb216d42..cedd5b97ed6 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java @@ -471,7 +471,7 @@ public abstract class AsynchronousSectionedRenderer e stream.close(); } catch (IOException e) { closeException = e; - logger.log(LogLevel.WARNING, "Exception caught while closing stream to client.", e); + logger.log(Level.WARNING, "Exception caught while closing stream to client.", e); } finally { if (failed != null) { success.setException(failed); diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 5b8f7b8b106..af473dd7628 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -168,7 +168,7 @@ public class StemmingSearcher extends Searcher { if (i instanceof TermItem) { return ((TermItem) i).getOrigin(); // this should always be the case } else { - getLogger().log(LogLevel.WARNING, "Weird, BlockItem '" + b + "' was a composite containing " + + getLogger().log(Level.WARNING, "Weird, BlockItem '" + b + "' was a composite containing " + i.getClass().getName() + ", expected TermItem."); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 8e83782c9c7..b4b23a0e94a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -243,7 +243,7 @@ public class StatisticsSearcher extends Searcher { if (latency >= 0) { addLatency(latency, metricContext); } else { - getLogger().log(LogLevel.WARNING, + getLogger().log(Level.WARNING, "Apparently negative latency measure, start: " + start + ", end: " + end + ", for query: " + query.toString()); } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java index d30464e349e..3abfa3a6531 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java @@ -110,7 +110,7 @@ public class UniqueGroupingSearcher extends Searcher { if (null == root) { String msg = "Result group not found for deduping grouping request, returning empty result."; query.trace(msg, 3); - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); throw new IllegalStateException("Failed to produce deduped result set."); } result.hits().remove(root.getId().toString()); // hide our tracks diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 2a8eeaf8e7f..432b378938e 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -417,7 +417,7 @@ public class SearchHandler extends LoggingRequestHandler { if (searchConnections != null) { connectionStatistics(); } else { - log.log(LogLevel.WARNING, + log.log(Level.WARNING, "searchConnections is a null reference, probably a known race condition during startup.", new IllegalStateException("searchConnections reference is null.")); } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java index d678b4d2487..d79122ac9c4 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java @@ -395,7 +395,7 @@ public final class SyncDefaultRenderer extends Renderer { if (e instanceof IOException) { throw (IOException) e; } else { - log.log(LogLevel.WARNING, "Exception thrown when rendering the result:", e); + log.log(Level.WARNING, "Exception thrown when rendering the result:", e); } } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java index e9de02b243b..230af8971c2 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java @@ -17,7 +17,7 @@ public class LoggingTraceExporter implements TraceExporter { public void maybeExport(Supplier traceDescriptionSupplier) { var traceDescription = traceDescriptionSupplier.get(); if (traceDescription.getTrace() != null) { - log.log(LogLevel.WARNING, String.format("%s: %s", traceDescription.getDescription(), + log.log(Level.WARNING, String.format("%s: %s", traceDescription.getDescription(), traceDescription.getTrace().toString())); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index a1b97dad56a..7e0fec4ba66 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -251,10 +251,10 @@ public class EndpointCertificateManager { // Normally because the cert is in the process of being provisioned - this will cause a retry in InternalStepRunner throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Certificate not found in secret store"); } catch (EndpointCertificateException e) { - log.log(LogLevel.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e); + log.log(Level.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e); throw e; } catch (Exception e) { - log.log(LogLevel.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e); + log.log(Level.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e); throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate validation failure for app " + instance.id().serializedForm(), e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 56d3b928b8f..e2fb74ec0b4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -69,7 +69,7 @@ public class DeploymentTrigger { public void notifyOfSubmission(TenantAndApplicationId id, ApplicationVersion version, long projectId) { if (applications().getApplication(id).isEmpty()) { - log.log(LogLevel.WARNING, "Ignoring submission from project '" + projectId + + log.log(Level.WARNING, "Ignoring submission from project '" + projectId + "': Unknown application '" + id + "'"); return; } @@ -117,7 +117,7 @@ public class DeploymentTrigger { */ public void notifyOfCompletion(ApplicationId id) { if (applications().getInstance(id).isEmpty()) { - log.log(LogLevel.WARNING, "Ignoring completion of job of unknown application '" + id + "'"); + log.log(Level.WARNING, "Ignoring completion of job of unknown application '" + id + "'"); return; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index 6f60106c79e..855cbdb5337 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -74,7 +74,7 @@ public class NameServiceForwarder { NameServiceQueue queue = db.readNameServiceQueue(); var queued = queue.requests().size(); if (queued >= maxQueuedRequests) { - log.log(LogLevel.WARNING, "Queue is at capacity (size: " + queued + "), dropping older " + + log.log(Level.WARNING, "Queue is at capacity (size: " + queued + "), dropping older " + "requests. This likely means that the name service is not successfully " + "executing requests"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java index aebf4a61966..ee5d50414c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java @@ -78,7 +78,7 @@ public class NameServiceQueue { request.dispatchTo(nameService); queue.requests.poll(); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to execute " + request + ": " + e.getMessage() + + log.log(Level.WARNING, "Failed to execute " + request + ": " + e.getMessage() + ", request will be retried"); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index 8346fe7a0f9..12b52d1bf2f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -55,7 +55,7 @@ public class ContactInformationMaintainer extends Maintainer { throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to update contact information for " + tenant + ": " + + log.log(Level.WARNING, "Failed to update contact information for " + tenant + ": " + Exceptions.toMessageString(e) + ". Retrying in " + maintenanceInterval()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 9843d808564..a4078f7e028 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -80,7 +80,7 @@ public class DeploymentMetricsMaintainer extends Maintainer { try { pool.awaitTermination(30, TimeUnit.MINUTES); if (lastException.get() != null) { - log.log(LogLevel.WARNING, + log.log(Level.WARNING, String.format("Failed to gather metrics for %d/%d applications. Retrying in %s. Last error: %s", failures.get(), attempts.get(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index d78091b5116..e8391b8aded 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -86,7 +86,7 @@ public class JobRunner extends Maintainer { .ifPresent(run -> controller().applications().deploymentTrigger().notifyOfCompletion(id.application())); } catch (Exception e) { - log.log(LogLevel.WARNING, "Exception finishing " + id, e); + log.log(Level.WARNING, "Exception finishing " + id, e); } } @@ -118,7 +118,7 @@ public class JobRunner extends Maintainer { // Something else is already advancing this step, or a prerequisite -- try again later! } catch (RuntimeException e) { - log.log(LogLevel.WARNING, "Exception attempting to advance " + step + " of " + id, e); + log.log(Level.WARNING, "Exception attempting to advance " + step + " of " + id, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index e08ed6a6178..4b4d2c8698d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -23,7 +23,7 @@ public class OsVersionStatusUpdater extends Maintainer { OsVersionStatus newStatus = OsVersionStatus.compute(controller()); controller().updateOsVersionStatus(newStatus); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + + log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + maintenanceInterval()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java index 0253cdb1e34..68f8e5464a0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java @@ -68,7 +68,7 @@ public class RotationStatusUpdater extends Maintainer { try { pool.awaitTermination(30, TimeUnit.SECONDS); if (lastException.get() != null) { - log.log(LogLevel.WARNING, String.format("Failed to get global routing status of %d/%d applications. Retrying in %s. Last error: ", + log.log(Level.WARNING, String.format("Failed to get global routing status of %d/%d applications. Retrying in %s. Last error: ", failures.get(), attempts.get(), maintenanceInterval()), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index e678c4bb5ca..22030931944 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -27,7 +27,7 @@ public class VersionStatusUpdater extends Maintainer { VersionStatus newStatus = VersionStatus.compute(controller()); controller().updateVersionStatus(newStatus); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + + log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + maintenanceInterval()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 757ccbe900e..2b9fe708df6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -52,7 +52,7 @@ public class ControllerAuthorizationFilter extends JsonSecurityRequestFilterBase return Optional.empty(); } catch (Exception e) { - log.log(LogLevel.WARNING, "Exception evaluating access control: ", e); + log.log(Level.WARNING, "Exception evaluating access control: ", e); } return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Access denied")); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java index b4fa9d261c1..538ab36374b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java @@ -85,7 +85,7 @@ class SystemFlagsDeployer { currentFlagData = lookupTable(client.listFlagData(target)); definedFlags = client.listDefinedFlags(target); } catch (Exception e) { - log.log(LogLevel.WARNING, String.format("Failed to list flag data for target '%s': %s", target, e.getMessage()), e); + log.log(Level.WARNING, String.format("Failed to list flag data for target '%s': %s", target, e.getMessage()), e); return new SystemFlagsDeployResult(List.of(OperationError.listFailed(e.getMessage(), target))); } @@ -119,7 +119,7 @@ class SystemFlagsDeployer { dryRunFlagDataValidation(data); } } catch (Exception e) { - log.log(LogLevel.WARNING, String.format("Failed to put flag '%s' for target '%s': %s", data.id(), target, e.getMessage()), e); + log.log(Level.WARNING, String.format("Failed to put flag '%s' for target '%s': %s", data.id(), target, e.getMessage()), e); errors.add(OperationError.createFailed(e.getMessage(), target, data)); return; } @@ -145,7 +145,7 @@ class SystemFlagsDeployer { dryRunFlagDataValidation(wantedData); } } catch (Exception e) { - log.log(LogLevel.WARNING, String.format("Failed to update flag '%s' for target '%s': %s", wantedData.id(), target, e.getMessage()), e); + log.log(Level.WARNING, String.format("Failed to update flag '%s' for target '%s': %s", wantedData.id(), target, e.getMessage()), e); errors.add(OperationError.updateFailed(e.getMessage(), target, wantedData)); return; } @@ -167,7 +167,7 @@ class SystemFlagsDeployer { try { client.deleteFlagData(target, id); } catch (Exception e) { - log.log(LogLevel.WARNING, String.format("Failed to delete flag '%s' for target '%s': %s", id, target, e.getMessage()), e); + log.log(Level.WARNING, String.format("Failed to delete flag '%s' for target '%s': %s", id, target, e.getMessage()), e); errors.add(OperationError.deleteFailed(e.getMessage(), target, id)); return; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 41a3784163d..19b98ae12cc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -151,7 +151,7 @@ public class VersionStatus { if (nodes.isEmpty()) continue; var configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty()); if (!configConverged) { - log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone.getId() + + log.log(Level.WARNING, "Config for " + application.id() + " in " + zone.getId() + " has not converged"); } for (var node : nodes) { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index 810e8c6ada4..022c8254870 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -60,7 +60,7 @@ public class DocumentProcessingTask implements Runnable { processings.addAll(requestContext.getProcessings()); } catch (Exception e) { //deserialization failed: - log.log(LogLevel.WARNING, "Deserialization of message failed.", e); + log.log(Level.WARNING, "Deserialization of message failed.", e); requestContext.processingFailed(e); return; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index c2c36d01805..b40da64df30 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -467,7 +467,7 @@ public class MessageBusVisitorSession implements VisitorSession { // we just immediately go into a failure destruction mode as soon as this // happens, in which we do not wait for any active messages to be replied // to. - log.log(LogLevel.WARNING, "Visitor session '" + sessionName + + log.log(Level.WARNING, "Visitor session '" + sessionName + "': failed to submit reply task to executor service! " + "Session cannot reliably continue; terminating it early.", e); @@ -661,7 +661,7 @@ public class MessageBusVisitorSession implements VisitorSession { String msg = "Got exception of type " + e.getClass().getName() + " with message '" + e.getMessage() + "' while attempting to send visitors"; - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); transitionTo(new StateDescription(State.FAILED, msg)); // It's likely that the exception caused a failure to send a // visitor message, meaning we won't get a reply task in the diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java index f89ab28ee16..12dea1759c4 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java @@ -148,7 +148,7 @@ public class StoragePolicy extends SlobrokPolicy { if (arr.size() == 1) { return convertSlobrokNameToSessionName(arr.get(0).getSpecString()); } else { - log.log(LogLevel.WARNING, "Got " + arr.size() + " matches for a distributor."); + log.log(Level.WARNING, "Got " + arr.size() + " matches for a distributor."); } } else { return convertSlobrokNameToSessionName(arr.get(randomizer.nextInt(arr.size())).getSpecString()); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java index b13a273da27..ee851acc0f8 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java @@ -38,7 +38,7 @@ public class SubsetServicePolicy implements DocumentProtocolRoutingPolicy { subsetSize = Integer.parseInt(param); } catch (NumberFormatException e) { - log.log(LogLevel.WARNING, "Parameter '" + param + "' could not be parsed as an integer.", e); + log.log(Level.WARNING, "Parameter '" + param + "' could not be parsed as an integer.", e); } if (subsetSize <= 0) { log.warning("Ignoring a request to set the subset size to " + subsetSize + " because it makes no " + diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java index 66b713ac6ba..0ed72076549 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java @@ -80,14 +80,14 @@ public class CompressedFileReference { if (!(outFile.exists() && outFile.isDirectory())) { log.log(Level.FINE, () -> "Creating dir: " + outFile.getAbsolutePath()); if (!outFile.mkdirs()) { - log.log(LogLevel.WARNING, "Could not create dir " + entry.getName()); + log.log(Level.WARNING, "Could not create dir " + entry.getName()); } } } else { // Create parent dir if necessary File parent = new File(outFile.getParent()); if (!parent.exists() && !parent.mkdirs()) { - log.log(LogLevel.WARNING, "Could not create dir " + parent.getAbsolutePath()); + log.log(Level.WARNING, "Could not create dir " + parent.getAbsolutePath()); } FileOutputStream fos = new FileOutputStream(outFile); ByteStreams.copy(archiveInputStream, fos); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index cb3a3caf2d4..a68d2860874 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -51,7 +51,7 @@ public class FileDownloader { try { return getFutureFile(fileReferenceDownload).get(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { - log.log(LogLevel.WARNING, "Failed downloading '" + fileReferenceDownload.fileReference().value() + "', removing from download queue: " + e.getMessage()); + log.log(Level.WARNING, "Failed downloading '" + fileReferenceDownload.fileReference().value() + "', removing from download queue: " + e.getMessage()); fileReferenceDownloader.failedDownloading(fileReferenceDownload.fileReference()); return Optional.empty(); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 4af10b69be6..b18b415931b 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -210,7 +210,7 @@ public class FileReceiver { else Files.delete(path.toPath()); } catch (IOException ioe) { - log.log(LogLevel.WARNING, "Failed deleting file/dir " + path); + log.log(Level.WARNING, "Failed deleting file/dir " + path); } } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 35f8636c0f9..11dd5195ee0 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -155,7 +155,7 @@ public class FileReferenceDownloader { } else if (request.returnValues().size() == 0) { return false; } else if (!request.checkReturnTypes("is")) { // TODO: Do not hard-code return type - log.log(LogLevel.WARNING, "Invalid return types for response: " + request.errorMessage()); + log.log(Level.WARNING, "Invalid return types for response: " + request.errorMessage()); return false; } return true; diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java index 8045e35604a..922e4140868 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java @@ -86,7 +86,7 @@ public final class MbusClient extends AbstractResource implements ClientProvider handler.handleResponse(new MbusResponse(StatusCodes.fromMbusReply(reply), reply)) .close(IgnoredCompletionHandler.INSTANCE); } catch (final Exception e) { - log.log(LogLevel.WARNING, "Ignoring exception thrown by ResponseHandler.", e); + log.log(Level.WARNING, "Ignoring exception thrown by ResponseHandler.", e); } } @@ -139,7 +139,7 @@ public final class MbusClient extends AbstractResource implements ClientProvider } sendBlocking(request); } catch (final Exception e) { - log.log(LogLevel.WARNING, "Ignoring exception thrown by MbusClient.", e); + log.log(Level.WARNING, "Ignoring exception thrown by MbusClient.", e); } } } diff --git a/logserver/src/main/java/com/yahoo/logserver/Flusher.java b/logserver/src/main/java/com/yahoo/logserver/Flusher.java index b155c8eb893..21af3d77f34 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Flusher.java +++ b/logserver/src/main/java/com/yahoo/logserver/Flusher.java @@ -51,7 +51,7 @@ public class Flusher extends Thread { } } } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "flusher was interrupted", e); + log.log(Level.WARNING, "flusher was interrupted", e); } } } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java index dc457d81ed5..da19edb8413 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java @@ -215,7 +215,7 @@ public class LogMetricsHandler extends AbstractLogHandler { try { Thread.sleep(EVENTINTERVAL * 1000); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, e.getMessage()); + log.log(Level.WARNING, e.getMessage()); } sendEvents(); } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java b/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java index 8fe3072cba9..b6d41a0f112 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java @@ -440,11 +440,11 @@ public class TraceNode implements Comparable { node.setStrict(c == '('); } else if (c == ')' || c == '}') { if (node == null) { - log.log(LogLevel.WARNING, "Unexpected closing brace in trace '" + str + "' at position " + i + "."); + log.log(Level.WARNING, "Unexpected closing brace in trace '" + str + "' at position " + i + "."); return new TraceNode(); } if (node.isStrict() != (c == ')')) { - log.log(LogLevel.WARNING, "Mismatched closing brace in trace '" + str + "' at position " + i + "."); + log.log(Level.WARNING, "Mismatched closing brace in trace '" + str + "' at position " + i + "."); return new TraceNode(); } node = node.parent; @@ -452,15 +452,15 @@ public class TraceNode implements Comparable { } } if (note != null) { - log.log(LogLevel.WARNING, "Unterminated note in trace '" + str + "'."); + log.log(Level.WARNING, "Unterminated note in trace '" + str + "'."); return new TraceNode(); } if (node != proxy) { - log.log(LogLevel.WARNING, "Missing closing brace in trace '" + str + "'."); + log.log(Level.WARNING, "Missing closing brace in trace '" + str + "'."); return new TraceNode(); } if (proxy.getNumChildren() == 0) { - log.log(LogLevel.WARNING, "No nodes found in trace '" + str + "'."); + log.log(Level.WARNING, "No nodes found in trace '" + str + "'."); return new TraceNode(); } if (proxy.getNumChildren() != 1) { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java index c1029175080..25a5253d7be 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java @@ -141,7 +141,7 @@ public class RPCTarget implements RequestWaiter { log.log(Level.FINE, "Target '" + name + "' has version " + version + "."); } } catch (IllegalArgumentException e) { - log.log(LogLevel.WARNING, "Failed to parse '" + str + "' as version for target '" + name + "'.", e); + log.log(Level.WARNING, "Failed to parse '" + str + "' as version for target '" + name + "'.", e); } } else { log.log(Level.INFO, "Method mbus.getVersion() failed for target '" + name + "'; " + diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java index 06036aa90d5..17a6fb234c0 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java @@ -166,10 +166,10 @@ public class ConfigSentinelClient extends AbstractComponent { { servicelist = req.returnValues().get(0).asString(); } else { - log.log(LogLevel.WARNING, "Bad answer to RPC request: " + req.errorMessage()); + log.log(Level.WARNING, "Bad answer to RPC request: " + req.errorMessage()); } } else { - log.log(LogLevel.WARNING, "Could not connect to sentinel at: "+spec); + log.log(Level.WARNING, "Could not connect to sentinel at: "+spec); } return servicelist; } finally { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index 905682df889..d1e1d6f14cc 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -72,7 +72,7 @@ public abstract class HttpMetricFetcher { void logMessageNoResponse(String message, int timesFetched) { if (timesFetched > 5) { - log.log(LogLevel.WARNING, message); + log.log(Level.WARNING, message); } else { log.log(Level.INFO, message); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index ca2e7d192dd..453f4f4b4c1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -180,9 +180,9 @@ public class NodeAdminStateUpdater { .collect(Collectors.toSet()); nodeAdmin.refreshContainersToRun(nodeAgentContexts); } catch (ConvergenceException e) { - log.log(LogLevel.WARNING, "Failed to update which containers should be running: " + Exceptions.toMessageString(e)); + log.log(Level.WARNING, "Failed to update which containers should be running: " + Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(LogLevel.WARNING, "Failed to update which containers should be running", e); + log.log(Level.WARNING, "Failed to update which containers should be running", e); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 12628933e9a..d473b3dcf40 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -568,7 +568,7 @@ public class NodeAgentImpl implements NodeAgent { try { aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); } catch (RuntimeException suppressed) { - logger.log(LogLevel.WARNING, "Suppressing ACL update failure: " + suppressed); + logger.log(Level.WARNING, "Suppressing ACL update failure: " + suppressed); e.addSuppressed(suppressed); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java index 577ff53ebdf..668bffd570a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java @@ -92,7 +92,7 @@ public class ChildProcess2Impl implements ChildProcess2 { if (commandLine.getOutputFile().isEmpty()) Files.delete(outputPath); } catch (Throwable t) { - logger.log(LogLevel.WARNING, "Failed to delete " + outputPath, t); + logger.log(Level.WARNING, "Failed to delete " + outputPath, t); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java index 4bab918c5a7..f4be8b7479f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java @@ -88,7 +88,7 @@ public class ProcessFactoryImpl implements ProcessFactory { if (commandLine.getOutputFile().isEmpty()) Files.delete(outputFile); } catch (IOException ioException) { - logger.log(LogLevel.WARNING, "Failed to delete temporary file at " + + logger.log(Level.WARNING, "Failed to delete temporary file at " + outputFile, ioException); } throw throwable; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java index 00856e6653c..b4c0d469272 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java @@ -42,7 +42,7 @@ public class CapacityReportMaintainer extends Maintainer { CapacityChecker capacityChecker = new CapacityChecker(this.nodeRepository); List overcommittedHosts = capacityChecker.findOvercommittedHosts(); if (overcommittedHosts.size() != 0) { - log.log(LogLevel.WARNING, String.format("%d nodes are overcommitted! [ %s ]", overcommittedHosts.size(), + log.log(Level.WARNING, String.format("%d nodes are overcommitted! [ %s ]", overcommittedHosts.size(), overcommittedHosts.stream().map(Node::hostname).collect(Collectors.joining(", ")))); } metric.set("overcommittedHosts", overcommittedHosts.size(), null); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index e6b1823f830..55152f52744 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -79,7 +79,7 @@ public class LoadBalancerExpirer extends Maintainer { } }); if (!failed.isEmpty()) { - log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s", + log.log(Level.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s", failed.size(), failed.stream() .map(LoadBalancerId::serializedForm) @@ -107,7 +107,7 @@ public class LoadBalancerExpirer extends Maintainer { } }); if (!failed.isEmpty()) { - log.log(LogLevel.WARNING, String.format("Failed to remove reals from %d load balancers: %s, retrying in %s", + log.log(Level.WARNING, String.format("Failed to remove reals from %d load balancers: %s, retrying in %s", failed.size(), failed.stream() .map(LoadBalancerId::serializedForm) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java index 6145a696af8..3319db33730 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java @@ -88,7 +88,7 @@ class MaintenanceDeployment implements Closeable { return false; } catch (RuntimeException e) { metric.add("maintenanceDeployment.failure", 1, metric.createContext(Map.of())); - log.log(LogLevel.WARNING, "Exception on maintenance deploy of " + application, e); + log.log(Level.WARNING, "Exception on maintenance deploy of " + application, e); return false; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 590847b8b7e..c33a61b0ea3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -155,7 +155,7 @@ public class NodePrioritizer { allocation = host.ipConfig().pool().findAllocation(allNodes, nameResolver); if (allocation.isEmpty()) continue; // No free addresses in this pool } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed allocating IP address on " + host.hostname(), e); + log.log(Level.WARNING, "Failed allocating IP address on " + host.hostname(), e); continue; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java index 7632bcb8936..1a8105e682f 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java @@ -82,7 +82,7 @@ class ZkApplicationLock implements ApplicationLock { onClose.run(); } catch (RuntimeException e) { // We may want to avoid logging some exceptions that may be expected, like when session expires. - log.log(LogLevel.WARNING, + log.log(Level.WARNING, "Failed close application lock in " + ZkApplicationLock.class.getSimpleName() + ", will ignore and continue", e); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java index 21c3c846385..a851781660a 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java @@ -195,7 +195,7 @@ public class ZkStatusService implements StatusService { lock.close(); } catch (RuntimeException e) { // We may want to avoid logging some exceptions that may be expected, like when session expires. - log.log(LogLevel.WARNING, + log.log(Level.WARNING, "Failed to close application lock for " + ZkStatusService.class.getSimpleName() + ", will ignore and continue", e); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index dea80254b05..c9df077aeef 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -132,7 +132,7 @@ public class DuperModel { // If an activation contains a host that is currently assigned to a // different application we will patch up our data structures to remain // internally consistent. But listeners may be fooled. - logger.log(LogLevel.WARNING, hostname + " has been reassigned from " + + logger.log(Level.WARNING, hostname + " has been reassigned from " + previousId.toFullString() + " to " + id.toFullString()); Set previousHostnames = hostnamesById.get(previousId); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java index dcaba9d6755..39aeb94667c 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java @@ -62,7 +62,7 @@ public class RunletExecutorImpl implements RunletExecutor { try { executor.awaitTermination(10, TimeUnit.MINUTES); } catch (InterruptedException e) { - logger.log(LogLevel.WARNING, "Timed out waiting for termination of executor", e); + logger.log(Level.WARNING, "Timed out waiting for termination of executor", e); } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java index f13d2ee006a..dd70c8d6992 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java @@ -68,7 +68,7 @@ public class StateV1HealthClient implements AutoCloseable { try { httpClient.close(); } catch (Exception e) { - logger.log(LogLevel.WARNING, "Failed to close CloseableHttpClient", e); + logger.log(Level.WARNING, "Failed to close CloseableHttpClient", e); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index 3c6ff8bd903..2fa97d41284 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -64,7 +64,7 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { synchronized (monitor) { SlobrokMonitor slobrokMonitor = slobrokMonitors.remove(id); if (slobrokMonitor == null) { - logger.log(LogLevel.WARNING, "Removed application " + id + + logger.log(Level.WARNING, "Removed application " + id + ", but it was never registered"); } else { slobrokMonitor.close(); diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java index 1a82b0ff42f..b75a0529a03 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java @@ -78,7 +78,7 @@ public class Bucket { get(entry.getKey(), entry.getValue()).merge(entry.getValue(), otherIsNewer); } } catch (IllegalArgumentException e) { - log.log(LogLevel.WARNING, "Problems merging metric " + metricName + ", possibly ignoring data."); + log.log(Level.WARNING, "Problems merging metric " + metricName + ", possibly ignoring data."); // avoid spamming the log if there are a lot of mismatching // threads malformedMetrics.add(metricName); diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java index 0a085b0c9ab..f757ab15022 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java @@ -96,7 +96,7 @@ public class UntypedMetric { try { histogram.add(other.histogram); } catch (ArrayIndexOutOfBoundsException e) { - log.log(LogLevel.WARNING, "Had trouble merging histograms: " + e.getMessage()); + log.log(Level.WARNING, "Had trouble merging histograms: " + e.getMessage()); } } } diff --git a/statistics/src/main/java/com/yahoo/statistics/Value.java b/statistics/src/main/java/com/yahoo/statistics/Value.java index 4260093fb20..8a5dbdbd880 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Value.java +++ b/statistics/src/main/java/com/yahoo/statistics/Value.java @@ -360,7 +360,7 @@ public class Value extends Handle { throw new IllegalStateException("The Limits instance must be frozen."); } if (parameters.histogramId != HistogramType.REGULAR) { - log.log(LogLevel.WARNING, HISTOGRAM_TYPE_WARNING + " '" + name + "'"); + log.log(Level.WARNING, HISTOGRAM_TYPE_WARNING + " '" + name + "'"); } this.histogramId = HistogramType.REGULAR; this.histogram = parameters.limits; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index a6498386b9a..71a4c1a9954 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -305,7 +305,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen ? athenzCredentialsService.registerInstance() : athenzCredentialsService.updateCredentials(credentials.getIdentityDocument(), identitySslContext)); } catch (Throwable t) { - log.log(LogLevel.WARNING, "Failed to update credentials: " + t.getMessage(), t); + log.log(Level.WARNING, "Failed to update credentials: " + t.getMessage(), t); } } @@ -315,7 +315,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen Duration remainingLifetime = Duration.between(clock.instant(), expirationTime); metric.set(CERTIFICATE_EXPIRY_METRIC_NAME, remainingLifetime.getSeconds(), null); } catch (Throwable t) { - log.log(LogLevel.WARNING, "Failed to update metrics: " + t.getMessage(), t); + log.log(Level.WARNING, "Failed to update metrics: " + t.getMessage(), t); } } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index b45bd0f4e9f..ccd2d80efb2 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -239,7 +239,7 @@ public class RestApi extends LoggingRequestHandler { RestUri.apiErrorCodes.PARSER_ERROR); } catch (RuntimeException systemException) { - log.log(LogLevel.WARNING, "Internal runtime exception during Document V1 request handling", systemException); + log.log(Level.WARNING, "Internal runtime exception during Document V1 request handling", systemException); return Response.createErrorResponse(500, Exceptions.toMessageString(systemException), restUri, RestUri.apiErrorCodes.UNSPECIFIED); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index 6bb74e98fbd..859b763ab9d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -144,7 +144,7 @@ class ClientFeederV3 { } catch (InterruptedException e) { // NOP, just terminate } catch (Throwable e) { - log.log(LogLevel.WARNING, "Unhandled exception while feeding: " + Exceptions.toMessageString(e), e); + log.log(Level.WARNING, "Unhandled exception while feeding: " + Exceptions.toMessageString(e), e); } finally { replies.add(createOperationStatus("-", "-", ErrorCode.END_OF_FEED, false, null)); } @@ -183,7 +183,7 @@ class ClientFeederV3 { message = getNextMessage(operationId.get(), requestInputStream, settings); } catch (Exception e) { if (log.isLoggable(LogLevel.WARNING)) { - log.log(LogLevel.WARNING, Exceptions.toMessageString(e)); + log.log(Level.WARNING, Exceptions.toMessageString(e)); } metric.add(MetricNames.PARSE_ERROR, 1, null); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index 502ddcddc9c..05a64d633a4 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -116,11 +116,11 @@ public class FeedHandlerV3 extends LoggingRequestHandler { return clientFeederV3.handleRequest(request); } catch (UnknownClientException uce) { String msg = Exceptions.toMessageString(uce); - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); return new ErrorHttpResponse(com.yahoo.jdisc.http.HttpResponse.Status.BAD_REQUEST, msg); } catch (Exception e) { String msg = "Could not initialize document parsing: " + Exceptions.toMessageString(e); - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); return new ErrorHttpResponse(com.yahoo.jdisc.http.HttpResponse.Status.INTERNAL_SERVER_ERROR, msg); } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java index 7fb9d019e6c..5c0c4ffaf3d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -62,7 +62,7 @@ public class FeedReplyReader implements ReplyHandler { context.feedReplies.put(new OperationStatus(message, context.docId, status, isConditionNotMet, traceMessage)); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, + log.log(Level.WARNING, "Interrupted while enqueueing result from putting document with id: " + context.docId); Thread.currentThread().interrupt(); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java index aa2849a027b..6629ce5086a 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java @@ -139,7 +139,7 @@ public class SharedSender implements ReplyHandler { owner.getPending().clear(); } } else { - log.log(LogLevel.WARNING, "Received reply " + r + " for message " + r.getMessage() + " without context"); + log.log(Level.WARNING, "Received reply " + r + " for message " + r.getMessage() + " without context"); } } diff --git a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java index f3a2540b244..1f4be8852a3 100644 --- a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -165,7 +165,7 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna try { zkServerThread.join(); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "Error joining server thread on shutdown", e); + log.log(Level.WARNING, "Error joining server thread on shutdown", e); } } -- cgit v1.2.3 From 545c7069fbf76932542f80a25aab44fdc972925a Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 15:47:56 +0200 Subject: LogLevel.ERROR -> Level.SEVERE --- .../ConfigserverSslContextFactoryProvider.java | 2 +- .../identitydocument/IdentityDocumentResource.java | 2 +- .../instanceconfirmation/InstanceConfirmationResource.java | 2 +- .../instanceconfirmation/InstanceRefreshResource.java | 2 +- .../instanceconfirmation/InstanceValidator.java | 2 +- .../vespa/clustercontroller/core/FleetController.java | 4 ++-- .../core/database/MasterDataGatherer.java | 2 +- .../yahoo/vespa/clustercontroller/core/DummyVdsNode.java | 10 +++++----- .../proxy/filedistribution/UrlDownloadRpcServer.java | 6 +++--- .../yahoo/vespa/config/server/ConfigServerBootstrap.java | 8 ++++---- .../java/com/yahoo/vespa/config/server/ConfigServerDB.java | 2 +- .../config/server/filedistribution/FileDirectory.java | 2 +- .../vespa/config/server/http/v2/SessionActiveHandler.java | 2 +- .../vespa/config/server/modelfactory/ModelsBuilder.java | 2 +- .../yahoo/vespa/config/server/zookeeper/ConfigCurator.java | 2 +- .../yahoo/container/jdisc/ThreadedHttpRequestHandler.java | 2 +- .../com/yahoo/container/jdisc/ConfiguredApplication.java | 2 +- .../java/com/yahoo/prelude/fastsearch/DocsumField.java | 2 +- .../java/com/yahoo/prelude/query/parser/SpecialTokens.java | 2 +- .../main/java/com/yahoo/search/handler/SearchHandler.java | 4 ++-- .../vespa/hosted/controller/persistence/CuratorDb.java | 2 +- .../restapi/systemflags/SystemFlagsDeployer.java | 2 +- .../controller/restapi/systemflags/SystemFlagsHandler.java | 2 +- .../java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java | 2 +- .../com/yahoo/docproc/jdisc/DocumentProcessingHandler.java | 4 ++-- .../documentapi/messagebus/MessageBusVisitorSession.java | 10 +++++----- .../messagebus/protocol/RoutableRepository.java | 14 +++++++------- .../messagebus/protocol/RoutingPolicyRepository.java | 4 ++-- .../documentapi/messagebus/protocol/StoragePolicy.java | 4 ++-- .../documentapi/messagebus/systemstate/rule/NodeState.java | 2 +- .../com/yahoo/vespa/filedistribution/FileReceiver.java | 12 ++++++------ .../java/com/yahoo/logserver/AbstractPluginLoader.java | 4 ++-- .../yahoo/logserver/handlers/archive/ArchiverHandler.java | 2 +- .../src/main/java/com/yahoo/messagebus/Messenger.java | 2 +- .../main/java/com/yahoo/messagebus/ProtocolRepository.java | 6 +++--- .../vespa/metricsproxy/service/ConfigSentinelClient.java | 4 ++-- .../java/ai/vespa/metricsproxy/service/SystemPoller.java | 4 ++-- .../com/yahoo/vespa/orchestrator/OrchestratorContext.java | 2 +- .../yahoo/vespa/orchestrator/status/ZkStatusService.java | 4 ++-- .../com/yahoo/vespa/service/executor/CancellableImpl.java | 2 +- 40 files changed, 75 insertions(+), 75 deletions(-) diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index a4d9ded9254..ae4a5933ac2 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -174,7 +174,7 @@ public class ConfigserverSslContextFactoryProvider extends TlsContextBasedProvid keyManager.updateKeystore(keyStore, keystorePwd); log.log(Level.INFO, "Certificate successfully updated"); } catch (Throwable t) { - log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + t.getMessage(), t); + log.log(Level.SEVERE, "Failed to update certificate from ZTS: " + t.getMessage(), t); } } } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java index b49a35f0506..cb0b1e0557f 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java @@ -43,7 +43,7 @@ public class IdentityDocumentResource implements IdentityDocumentApi { return EntityBindingsMapper.toSignedIdentityDocumentEntity(identityDocumentGenerator.generateSignedIdentityDocument(hostname, identityType)); } catch (Exception e) { String message = String.format("Unable to generate identity doument for '%s': %s", hostname, e.getMessage()); - log.log(LogLevel.ERROR, message, e); + log.log(Level.SEVERE, message, e); throw new InternalServerErrorException(message, e); } } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java index 2828880ff67..dbe26d3a6bb 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java @@ -34,7 +34,7 @@ public class InstanceConfirmationResource { public InstanceConfirmation confirmInstance(InstanceConfirmation instanceConfirmation) { log.log(Level.FINE, instanceConfirmation.toString()); if (!instanceValidator.isValidInstance(instanceConfirmation)) { - log.log(LogLevel.ERROR, "Invalid instance: " + instanceConfirmation); + log.log(Level.SEVERE, "Invalid instance: " + instanceConfirmation); throw new ForbiddenException("Instance is invalid"); } return instanceConfirmation; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java index a1eab136aa2..b49e03658e6 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java @@ -36,7 +36,7 @@ public class InstanceRefreshResource { public InstanceConfirmation confirmInstanceRefresh(InstanceConfirmation instanceConfirmation) { log.log(Level.FINE, instanceConfirmation.toString()); if (!instanceValidator.isValidRefresh(instanceConfirmation)) { - log.log(LogLevel.ERROR, "Invalid instance refresh: " + instanceConfirmation); + log.log(Level.SEVERE, "Invalid instance refresh: " + instanceConfirmation); throw new ForbiddenException("Instance is invalid"); } return instanceConfirmation; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index cfc63df4a92..05bbd790ad7 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -92,7 +92,7 @@ public class InstanceValidator { log.log(Level.FINE, () -> String.format("Instance %s is valid.", providerUniqueId)); return true; } - log.log(LogLevel.ERROR, () -> String.format("Instance %s has invalid signature.", providerUniqueId)); + log.log(Level.SEVERE, () -> String.format("Instance %s has invalid signature.", providerUniqueId)); return false; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index b9c839c3b87..658dd10f7e5 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -652,7 +652,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd try { propagateOptions(); } catch (Exception e) { - log.log(LogLevel.ERROR, "Failed to handle new fleet controller config", e); + log.log(Level.SEVERE, "Failed to handle new fleet controller config", e); } } @@ -1070,7 +1070,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd log.log(Level.FINE, "Event thread stopped by interrupt exception: " + e); } catch (Throwable t) { t.printStackTrace(); - log.log(LogLevel.ERROR, "Fatal error killed fleet controller", t); + log.log(Level.SEVERE, "Fatal error killed fleet controller", t); synchronized (monitor) { running.set(false); } System.exit(1); } finally { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java index 75dfb90d059..cd5202c89b2 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java @@ -121,7 +121,7 @@ public class MasterDataGatherer { if (nextMasterData.containsKey(index)) { nextMasterData.remove(index); } else { - log.log(LogLevel.ERROR, "Fleetcontroller " + nodeIndex + ": Strangely, we already had data from node " + index + " when trying to remove it"); + log.log(Level.SEVERE, "Fleetcontroller " + nodeIndex + ": Strangely, we already had data from node " + index + " when trying to remove it"); } } else { Integer value = Integer.valueOf(data); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java index a596df164af..0cf84cfe54c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java @@ -468,7 +468,7 @@ public class DummyVdsNode { } } } catch (Exception e) { - log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering " + req.methodName() + " request: " + e.getMessage()); + log.log(Level.SEVERE, "Dummy node " + this + ": An error occurred when answering " + req.methodName() + " request: " + e.getMessage()); e.printStackTrace(System.err); req.setError(ErrorCode.METHOD_FAILED, e.getMessage()); } @@ -528,7 +528,7 @@ public class DummyVdsNode { req.returnValues().add(new StringValue("OK")); log.log(Level.FINE, "Dummy node " + this + ": Got new system state (through old setsystemstate call) " + newState); } catch (Exception e) { - log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering setsystemstate request: " + e.getMessage()); + log.log(Level.SEVERE, "Dummy node " + this + ": An error occurred when answering setsystemstate request: " + e.getMessage()); e.printStackTrace(System.err); req.returnValues().add(new Int32Value(ErrorCode.METHOD_FAILED)); req.returnValues().add(new StringValue(e.getMessage())); @@ -552,7 +552,7 @@ public class DummyVdsNode { } log.log(Level.FINE, "Dummy node " + this + ": Got new system state " + newState); } catch (Exception e) { - log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering setsystemstate request: " + e.getMessage()); + log.log(Level.SEVERE, "Dummy node " + this + ": An error occurred when answering setsystemstate request: " + e.getMessage()); e.printStackTrace(System.err); req.setError(ErrorCode.METHOD_FAILED, e.getMessage()); } @@ -575,7 +575,7 @@ public class DummyVdsNode { } log.log(Level.FINE, "Dummy node " + this + ": Got new cluster state " + stateBundle); } catch (Exception e) { - log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering setdistributionstates request: " + e.getMessage()); + log.log(Level.SEVERE, "Dummy node " + this + ": An error occurred when answering setdistributionstates request: " + e.getMessage()); e.printStackTrace(System.err); req.setError(ErrorCode.METHOD_FAILED, e.getMessage()); } @@ -602,7 +602,7 @@ public class DummyVdsNode { } log.log(Level.FINE, "Dummy node " + this + ": Activating cluster state version " + activateVersion); } catch (Exception e) { - log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering activate_cluster_state_version request: " + e.getMessage()); + log.log(Level.SEVERE, "Dummy node " + this + ": An error occurred when answering activate_cluster_state_version request: " + e.getMessage()); e.printStackTrace(System.err); req.setError(ErrorCode.METHOD_FAILED, e.getMessage()); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java index 21baf873f6d..d51144a5715 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java @@ -87,12 +87,12 @@ class UrlDownloadRpcServer { req.returnValues().add(new StringValue(new File(downloadDir, CONTENTS_FILE_NAME).getAbsolutePath())); } else { - log.log(LogLevel.ERROR, "Download of URL '" + url + "' got server response: " + connection.getResponseCode()); + log.log(Level.SEVERE, "Download of URL '" + url + "' got server response: " + connection.getResponseCode()); req.setError(HTTP_ERROR, String.valueOf(connection.getResponseCode())); } } catch (Throwable e) { - log.log(LogLevel.ERROR, "Download of URL '" + url + "' got exception: " + e.getMessage()); + log.log(Level.SEVERE, "Download of URL '" + url + "' got exception: " + e.getMessage()); req.setError(INTERNAL_ERROR, "Download of URL '" + url + "' internal error: " + e.getMessage()); } req.returnRequest(); @@ -115,7 +115,7 @@ class UrlDownloadRpcServer { log.log(Level.INFO, String.format("Download of URL '%s' done in %.3f seconds", url, (System.currentTimeMillis() -start) / 1000.0)); } else { - log.log(LogLevel.ERROR, "Downloaded URL '" + url + "' not found, returning error"); + log.log(Level.SEVERE, "Downloaded URL '" + url + "' not found, returning error"); req.setError(DOES_NOT_EXIST, "Downloaded '" + url + "' not found"); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 36767e6fc5f..d628702758b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -139,7 +139,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable try { Thread.sleep(1000); } catch (InterruptedException e) { - log.log(LogLevel.ERROR, "Got interrupted", e); + log.log(Level.SEVERE, "Got interrupted", e); break; } } while (server.isRunning()); @@ -158,7 +158,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable versionState.saveNewVersion(); log.log(Level.INFO, "All applications redeployed successfully"); } catch (Exception e) { - log.log(LogLevel.ERROR, "Redeployment of applications failed", e); + log.log(Level.SEVERE, "Redeployment of applications failed", e); redeployingApplicationsFailed(); return; // Status will not be set to 'up' since we return here } @@ -193,7 +193,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable try { Thread.sleep(10); } catch (InterruptedException e) { - log.log(LogLevel.ERROR, "Got interrupted", e); + log.log(Level.SEVERE, "Got interrupted", e); break; } } @@ -218,7 +218,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable } while ( ! applicationsNotRedeployed.isEmpty() && Instant.now().isBefore(end)); if ( ! applicationsNotRedeployed.isEmpty()) { - log.log(LogLevel.ERROR, "Redeploying applications not finished after " + maxDurationOfRedeployment + + log.log(Level.SEVERE, "Redeploying applications not finished after " + maxDurationOfRedeployment + ", exiting, applications that failed redeployment: " + applicationsNotRedeployed); return false; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java index 6cf0ad5fab7..c788b73720d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java @@ -35,7 +35,7 @@ public class ConfigServerDB { try { initialize(configserverConfig.configModelPluginDir()); } catch (IllegalArgumentException e) { - log.log(LogLevel.ERROR, "Error initializing serverdb: " + e.getMessage()); + log.log(Level.SEVERE, "Error initializing serverdb: " + e.getMessage()); } catch (IOException e) { throw new RuntimeException("Unable to initialize server db", e); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index cfa648ce351..9f65c0e3c6e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -119,7 +119,7 @@ public class FileDirectory { File existingFile = destinationDir.toPath().resolve(source.getName()).toFile(); if ( ! existingFile.exists() || ! computeHash(existingFile).equals(hashOfFileToBeAdded)) { - log.log(LogLevel.ERROR, "Directory for file reference '" + fileReference.value() + + log.log(Level.SEVERE, "Directory for file reference '" + fileReference.value() + "' has content that does not match its hash, deleting everything in " + destinationDir.getAbsolutePath()); IOUtils.recursiveDeleteDir(destinationDir); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index 5289092e9ce..4cf59135026 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -55,7 +55,7 @@ public class SessionActiveHandler extends SessionHandler { // Overridden to make sure we are logging when this low-level handling of timeout happens @Override public void handleTimeout(Request request, ResponseHandler responseHandler) { - log.log(LogLevel.ERROR, "activate timed out for " + request.getUri(), new RuntimeException("activate timed out")); + log.log(Level.SEVERE, "activate timed out for " + request.getUri(), new RuntimeException("activate timed out")); super.handleTimeout(request, responseHandler); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index ebb399076ad..6c6b352361f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -199,7 +199,7 @@ public abstract class ModelsBuilder { if (allApplicationVersions.size() > 0 && allApplicationVersions.get(0).getModel().skipOldConfigModels(now)) log.log(Level.INFO, applicationId + ": Skipping old version (due to validation override)"); else { - log.log(LogLevel.ERROR, applicationId + ": Failed to build version " + version); + log.log(Level.SEVERE, applicationId + ": Failed to build version " + version); throw e; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java index 748223b13a3..d0668772748 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -310,7 +310,7 @@ public class ConfigCurator { curator.framework().checkExists().forPath("/dummy"); } catch (Exception e) { - log.log(LogLevel.ERROR, "Unable to contact ZooKeeper on " + curator.connectionSpec() + + log.log(Level.SEVERE, "Unable to contact ZooKeeper on " + curator.connectionSpec() + ". Please verify for all configserver nodes that " + "VESPA_CONFIGSERVERS points to the correct configserver(s), " + "the same configserver(s) as in services.xml, and that they are started. " + diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 2b4c1489313..164a27dbf54 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -83,7 +83,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler } catch (Exception e) { metric.add(UNHANDLED_EXCEPTIONS_METRIC, 1L, contextFor(request, Map.of("exception", e.getClass().getSimpleName()))); metric.add(RENDERING_ERRORS, 1, null); - log.log(LogLevel.ERROR, "Uncaught exception handling request", e); + log.log(Level.SEVERE, "Uncaught exception handling request", e); if (channel != null) { channel.setHttpResponse(null); channel.close(null); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index f7e03a2098e..2a72d0b3442 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -210,7 +210,7 @@ public final class ConfiguredApplication implements Application { Container.get().setupFileAcquirer(config.filedistributor()); Container.get().setupUrlDownloader(); } catch (Exception e) { - log.log(LogLevel.ERROR, "Caught exception when initializing server. Exiting.", e); + log.log(Level.SEVERE, "Caught exception when initializing server. Exiting.", e); Runtime.getRuntime().halt(1); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 10ecb8604ea..ef892585d21 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -58,7 +58,7 @@ public abstract class DocsumField { fieldFactory.put("xmlstring", XMLField.class); fieldFactory.put("tensor", TensorField.class); } catch (Exception e) { - log.log(LogLevel.ERROR, "Could not initialize docsum decoding properly.", e); + log.log(Level.SEVERE, "Could not initialize docsum decoding properly.", e); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java index c24a2824b95..95a374316de 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java @@ -66,7 +66,7 @@ public class SpecialTokens { // TODO put along with the global toLowerCase String asHigh = token.toUpperCase(Locale.ENGLISH); if (asLow.length() != token.length() || asHigh.length() != token.length()) { - log.log(LogLevel.ERROR, "Special token '" + token + "' has case sensitive length. Ignoring the token." + log.log(Level.SEVERE, "Special token '" + token + "' has case sensitive length. Ignoring the token." + " Please report this message in a bug to the Vespa team."); return false; } else { diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 432b378938e..c8386f3c75c 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -485,10 +485,10 @@ public class SearchHandler extends LoggingRequestHandler { private void log(String request, Query query, Throwable e) { // Attempted workaround for missing stack traces if (e.getStackTrace().length == 0) { - log.log(LogLevel.ERROR, "Failed executing " + query.toDetailString() + + log.log(Level.SEVERE, "Failed executing " + query.toDetailString() + " [" + request + "], received exception with no context", e); } else { - log.log(LogLevel.ERROR, "Failed executing " + query.toDetailString() + " [" + request + "]", e); + log.log(Level.SEVERE, "Failed executing " + query.toDetailString() + " [" + request + "]", e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index ef347c9be19..db0866de5b6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -357,7 +357,7 @@ public class CuratorDb { readApplication(id).ifPresent(applications::add); } catch (Exception e) { if (canFail) { - log.log(LogLevel.ERROR, "Failed to read application '" + id + "', this must be fixed through " + + log.log(Level.SEVERE, "Failed to read application '" + id + "', this must be fixed through " + "manual intervention", e); } else { throw e; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java index 538ab36374b..5ea277f5101 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java @@ -65,7 +65,7 @@ class SystemFlagsDeployer { try { results.add(future.get()); } catch (InterruptedException | ExecutionException e) { - log.log(LogLevel.ERROR, String.format("Failed to deploy flags for target '%s': %s", target, e.getMessage()), e); + log.log(Level.SEVERE, String.format("Failed to deploy flags for target '%s': %s", target, e.getMessage()), e); throw new RuntimeException(e); } }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java index 76c746cd6c7..43a03bf10f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java @@ -66,7 +66,7 @@ public class SystemFlagsHandler extends LoggingRequestHandler { return new JacksonJsonResponse<>(200, result.toWire()); } catch (Exception e) { String errorMessage = "System flags deploy failed: " + e.getMessage(); - log.log(LogLevel.ERROR, errorMessage, e); + log.log(Level.SEVERE, errorMessage, e); return ErrorResponse.internalServerError(errorMessage); } } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 3a92d005b79..eb795de89b6 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -345,7 +345,7 @@ public class DockerImpl implements Docker { @Override public void onError(Throwable throwable) { removeScheduledPoll(dockerImage); - logger.log(LogLevel.ERROR, "Could not download image " + dockerImage.asString(), throwable); + logger.log(Level.SEVERE, "Could not download image " + dockerImage.asString(), throwable); } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index 2ebc29a6b09..335fec8c5c0 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -24,7 +24,6 @@ import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.ResponseHandler; -import java.util.logging.Level; import com.yahoo.messagebus.jdisc.MbusRequest; import com.yahoo.processing.execution.chain.ChainRegistry; import com.yahoo.statistics.Statistics; @@ -33,6 +32,7 @@ import java.util.TimerTask; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; +import java.util.logging.Level; import static com.yahoo.component.chain.ChainsConfigurer.prepareChainRegistry; import static com.yahoo.component.chain.model.ChainsModelBuilder.buildFromConfig; @@ -167,7 +167,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { DocprocService service = docprocServiceRegistry.getComponent(serviceName); //No need to enqueue a task if the docproc chain is empty, just forward requestContext if (service == null) { - log.log(LogLevel.ERROR, "DocprocService for session '" + serviceName + + log.log(Level.SEVERE, "DocprocService for session '" + serviceName + "' not found, returning request '" + requestContext + "'."); requestContext.processingFailed(RequestContext.ErrorCode.ERROR_PROCESSING_FAILURE, "DocprocService " + serviceName + " not found."); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index b40da64df30..429c9bc0f51 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -726,7 +726,7 @@ public class MessageBusVisitorSession implements VisitorSession { } else { String msg = "Received reply we do not know how to handle: " + reply.getClass().getName(); - log.log(LogLevel.ERROR, msg); + log.log(Level.SEVERE, msg); transitionTo(new StateDescription(State.FAILED, msg)); } } catch (Exception e) { @@ -772,7 +772,7 @@ public class MessageBusVisitorSession implements VisitorSession { private void handleMessageProcessingException(Reply reply, Exception e, String what) { final String errorDesc = formatProcessingException(e, what); final String fullMsg = formatIdentifyingVisitorErrorString(errorDesc); - log.log(LogLevel.ERROR, fullMsg, e); + log.log(Level.SEVERE, fullMsg, e); int errorCode; synchronized (progress.getToken()) { if (!params.skipBucketsOnFatalErrors()) { @@ -841,7 +841,7 @@ public class MessageBusVisitorSession implements VisitorSession { msg.swapState(reply); if (params.getLocalDataHandler() == null) { - log.log(LogLevel.ERROR, sessionName + ": Got visitor data back to client with no local data destination."); + log.log(Level.SEVERE, sessionName + ": Got visitor data back to client with no local data destination."); reply.addError(new Error(ErrorCode.APP_FATAL_ERROR, "Visitor data with no local data destination")); receiver.reply(reply); return; @@ -1066,7 +1066,7 @@ public class MessageBusVisitorSession implements VisitorSession { progress.getIterator().setDistributionBitCount(stateBits); } } catch (Exception e) { - log.log(LogLevel.ERROR, "Failed to parse new system state string: " + log.log(Level.SEVERE, "Failed to parse new system state string: " + reply.getSystemState()); transitionTo(new StateDescription(State.FAILED, "Failed to parse cluster state '" + reply.getSystemState() + "'")); @@ -1170,7 +1170,7 @@ public class MessageBusVisitorSession implements VisitorSession { sender.destroy(); receiver.destroy(); } catch (Exception e) { - log.log(LogLevel.ERROR, "Caught exception destroying communication interfaces", e); + log.log(Level.SEVERE, "Caught exception destroying communication interfaces", e); } log.log(Level.FINE, sessionName + ": synchronous destroy() done"); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java index c91db403cbd..7fc1ad8fc84 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java @@ -51,11 +51,11 @@ final class RoutableRepository { */ Routable decode(DocumentTypeManager docMan, Version version, byte[] data) { if (data == null || data.length == 0) { - log.log(LogLevel.ERROR, "Received empty byte array for deserialization."); + log.log(Level.SEVERE, "Received empty byte array for deserialization."); return null; } if (version.getMajor() < 5) { - log.log(LogLevel.ERROR,"Can not decode anything from (version " + version + "). Only major version 5 and up supported."); + log.log(Level.SEVERE,"Can not decode anything from (version " + version + "). Only major version 5 and up supported."); return null; } DocumentDeserializer in = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data)); @@ -64,12 +64,12 @@ final class RoutableRepository { int type = in.getInt(null); RoutableFactory factory = getFactory(version, type); if (factory == null) { - log.log(LogLevel.ERROR,"No routable factory found for routable type " + type + " (version " + version + ")."); + log.log(Level.SEVERE,"No routable factory found for routable type " + type + " (version " + version + ")."); return null; } Routable ret = factory.decode(in, loadTypes); if (ret == null) { - log.log(LogLevel.ERROR,"Routable factory " + factory.getClass().getName() + " failed to deserialize " + + log.log(Level.SEVERE,"Routable factory " + factory.getClass().getName() + " failed to deserialize " + "routable of type " + type + " (version " + version + ").\nData = " + Arrays.toString(data)); return null; } @@ -90,18 +90,18 @@ final class RoutableRepository { int type = obj.getType(); RoutableFactory factory = getFactory(version, type); if (factory == null) { - log.log(LogLevel.ERROR,"No routable factory found for routable type " + type + " (version " + version + ")."); + log.log(Level.SEVERE,"No routable factory found for routable type " + type + " (version " + version + ")."); return new byte[0]; } if (version.getMajor() < 5) { - log.log(LogLevel.ERROR,"Can not encode routable type " + type + " (version " + version + "). Only major version 5 and up supported."); + log.log(Level.SEVERE,"Can not encode routable type " + type + " (version " + version + "). Only major version 5 and up supported."); return new byte[0]; } DocumentSerializer out= DocumentSerializerFactory.createHead(new GrowableByteBuffer(8192)); out.putInt(null, type); if (!factory.encode(obj, out)) { - log.log(LogLevel.ERROR, "Routable factory " + factory.getClass().getName() + " failed to serialize " + + log.log(Level.SEVERE, "Routable factory " + factory.getClass().getName() + " failed to serialize " + "routable of type " + type + " (version " + version + ")."); return new byte[0]; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java index 0b9498d798a..c9e3488ecb9 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java @@ -49,13 +49,13 @@ class RoutingPolicyRepository { RoutingPolicy createPolicy(String name, String param) { RoutingPolicyFactory factory = getFactory(name); if (factory == null) { - log.log(LogLevel.ERROR, "No routing policy factory found for name '" + name + "'."); + log.log(Level.SEVERE, "No routing policy factory found for name '" + name + "'."); return null; } DocumentProtocolRoutingPolicy ret = factory.createPolicy(param); if (ret == null) { - log.log(LogLevel.ERROR, "Routing policy factory " + factory.getClass().getName() + " failed to create a " + + log.log(Level.SEVERE, "Routing policy factory " + factory.getClass().getName() + " failed to create a " + "routing policy for parameter '" + name + "'."); return null; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java index 12dea1759c4..5594b9f9db4 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java @@ -282,7 +282,7 @@ public class StoragePolicy extends SlobrokPolicy { case DocumentProtocol.MESSAGE_CREATEVISITOR: return ((CreateVisitorMessage)msg).getBuckets().get(0); case DocumentProtocol.MESSAGE_REMOVELOCATION: return ((RemoveLocationMessage)msg).getBucketId(); default: - log.log(LogLevel.ERROR, "Message type '" + msg.getType() + "' not supported."); + log.log(Level.SEVERE, "Message type '" + msg.getType() + "' not supported."); return null; } } @@ -471,7 +471,7 @@ public class StoragePolicy extends SlobrokPolicy { if (context.usedState == null) { String msg = "Used state must be set as distributor is calculated. Bug."; reply.getTrace().trace(1, msg); - log.log(LogLevel.ERROR, msg); + log.log(Level.SEVERE, msg); } else if (newState.getVersion() == context.usedState.getVersion()) { String msg = "Message sent to distributor " + context.calculatedDistributor + " retrieved cluster state version " + newState.getVersion() + diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java index 527c94a4d2a..d23a0e580a0 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java @@ -91,7 +91,7 @@ public class NodeState { } if (arr[0].equals(NODE_PARENT)) { if (parent == null) { - log.log(LogLevel.ERROR, "Location string '" + key + "' requests a parent above the top-most node, " + + log.log(Level.SEVERE, "Location string '" + key + "' requests a parent above the top-most node, " + "returning self to avoid crash."); } return parent.getChild(arr[1], force); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index b18b415931b..875058b2eff 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -80,7 +80,7 @@ public class FileReceiver { inprogressFile = Files.createTempFile(tmpDirectory.toPath(), fileName, ".inprogress").toFile(); } catch (IOException e) { String msg = "Failed creating temp file for inprogress file for " + fileName + " in '" + tmpDirectory.toPath() + "': "; - log.log(LogLevel.ERROR, msg + e.getMessage(), e); + log.log(Level.SEVERE, msg + e.getMessage(), e); throw new RuntimeException(msg, e); } } @@ -96,7 +96,7 @@ public class FileReceiver { try { Files.write(inprogressFile.toPath(), part, StandardOpenOption.WRITE, StandardOpenOption.APPEND); } catch (IOException e) { - log.log(LogLevel.ERROR, "Failed writing to file (" + inprogressFile.toPath() + "): " + e.getMessage(), e); + log.log(Level.SEVERE, "Failed writing to file (" + inprogressFile.toPath() + "): " + e.getMessage(), e); inprogressFile.delete(); throw new RuntimeException("Failed writing to file (" + inprogressFile.toPath() + "): ", e); } @@ -120,14 +120,14 @@ public class FileReceiver { try { Files.createDirectories(fileReferenceDir.toPath()); } catch (IOException e) { - log.log(LogLevel.ERROR, "Failed creating directory (" + fileReferenceDir.toPath() + "): " + e.getMessage(), e); + log.log(Level.SEVERE, "Failed creating directory (" + fileReferenceDir.toPath() + "): " + e.getMessage(), e); throw new RuntimeException("Failed creating directory (" + fileReferenceDir.toPath() + "): ", e); } log.log(Level.FINE, () -> "Uncompressed file, moving to " + file.getAbsolutePath()); moveFileToDestination(inprogressFile, file); } } catch (IOException e) { - log.log(LogLevel.ERROR, "Failed writing file: " + e.getMessage(), e); + log.log(Level.SEVERE, "Failed writing file: " + e.getMessage(), e); throw new RuntimeException("Failed writing file: ", e); } finally { try { @@ -135,7 +135,7 @@ public class FileReceiver { Files.delete(inprogressFile.toPath()); } } catch (IOException e) { - log.log(LogLevel.ERROR, "Failed deleting " + inprogressFile.getAbsolutePath() + ": " + e.getMessage(), e); + log.log(Level.SEVERE, "Failed deleting " + inprogressFile.getAbsolutePath() + ": " + e.getMessage(), e); } } return file; @@ -197,7 +197,7 @@ public class FileReceiver { deleteFileOrDirectory(tempFile); } catch (IOException e) { String message = "Failed moving file '" + tempFile.getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'"; - log.log(LogLevel.ERROR, message, e); + log.log(Level.SEVERE, message, e); throw new RuntimeException(message, e); } } diff --git a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java index 4988357ee8c..f897933ebe7 100644 --- a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java @@ -24,7 +24,7 @@ public abstract class AbstractPluginLoader implements PluginLoader { try { plugin = pluginClass.getDeclaredConstructor().newInstance(); } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { - log.log(LogLevel.ERROR, pluginClass.getName() + ": load failed: " + e); + log.log(Level.SEVERE, pluginClass.getName() + ": load failed: " + e); throw new RuntimeException(e); } @@ -42,7 +42,7 @@ public abstract class AbstractPluginLoader implements PluginLoader { plugin.initPlugin(config); log.log(Level.FINE, pname + ": plugin loaded"); } catch (Exception e) { - log.log(LogLevel.ERROR, pname + ": init failed", e); + log.log(Level.SEVERE, pname + ": init failed", e); } } } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index ea6883cbe29..8ed6ec6de00 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -215,7 +215,7 @@ public class ArchiverHandler extends AbstractLogHandler { log.log(Level.FINE, "Using " + absoluteRootDir + " as root"); } else { if (! root.mkdirs()) { - log.log(LogLevel.ERROR, "Unable to create directory " + absoluteRootDir); + log.log(Level.SEVERE, "Unable to create directory " + absoluteRootDir); } else { log.log(Level.FINE, "Created root at " + absoluteRootDir); } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java index 5f2ff58f3fd..0cc8310937b 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java @@ -168,7 +168,7 @@ public class Messenger implements Runnable { try { task.run(); } catch (final Exception e) { - log.log(LogLevel.ERROR, "An exception was thrown while running " + task.getClass().getName(), e); + log.log(Level.SEVERE, "An exception was thrown while running " + task.getClass().getName(), e); } try { task.destroy(); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java b/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java index d875cc32b3f..d3ab41331ae 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java @@ -78,20 +78,20 @@ public class ProtocolRepository { } Protocol protocol = getProtocol(protocolName); if (protocol == null) { - log.log(LogLevel.ERROR, "Protocol '" + protocolName + "' not supported."); + log.log(Level.SEVERE, "Protocol '" + protocolName + "' not supported."); return null; } try { ret = protocol.createPolicy(policyName, policyParam); } catch (RuntimeException e) { - log.log(LogLevel.ERROR, "Protcol '" + protocolName + "' threw an exception: " + e.getMessage(), e); + log.log(Level.SEVERE, "Protcol '" + protocolName + "' threw an exception: " + e.getMessage(), e); if (ret != null) { ret.destroy(); } return null; } if (ret == null) { - log.log(LogLevel.ERROR, "Protocol '" + protocolName + "' failed to create routing policy '" + policyName + + log.log(Level.SEVERE, "Protocol '" + protocolName + "' failed to create routing policy '" + policyName + "' with parameter '" + policyParam + "'."); return null; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java index 17a6fb234c0..c0e3e636aa6 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java @@ -45,7 +45,7 @@ public class ConfigSentinelClient extends AbstractComponent { try { setStatus(services); } catch (Exception e) { - log.log(LogLevel.ERROR, "Unable to update service pids from sentinel", e); + log.log(Level.SEVERE, "Unable to update service pids from sentinel", e); } } @@ -62,7 +62,7 @@ public class ConfigSentinelClient extends AbstractComponent { try { setStatus(services); } catch (Exception e) { - log.log(LogLevel.ERROR, "Unable to update service pids from sentinel", e); + log.log(Level.SEVERE, "Unable to update service pids from sentinel", e); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java index 7620e94fc9e..a6c2220b5a2 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java @@ -180,7 +180,7 @@ public class SystemPoller { try { in = new BufferedReader(new FileReader("/proc/stat")); } catch (FileNotFoundException ex) { - log.log(LogLevel.ERROR, "Unable to open stat file", ex); + log.log(Level.SEVERE, "Unable to open stat file", ex); return 0; } try { @@ -194,7 +194,7 @@ public class SystemPoller { in.close(); } catch (IOException ex) { - log.log(LogLevel.ERROR, "Unable to read line from stat file", ex); + log.log(Level.SEVERE, "Unable to read line from stat file", ex); return 0; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java index 1c5ae323d04..fbb25baa806 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java @@ -153,7 +153,7 @@ public class OrchestratorContext implements AutoCloseable { try { unlock.run(); } catch (RuntimeException e) { - logger.log(LogLevel.ERROR, "Failed run on close : " + e.getMessage()); + logger.log(Level.SEVERE, "Failed run on close : " + e.getMessage()); } }); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java index a851781660a..32fdaa9433d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java @@ -277,7 +277,7 @@ public class ZkStatusService implements StatusService { try { lock = lockApplication(context, reference); } catch (RuntimeException e) { - log.log(LogLevel.ERROR, "Failed to get Orchestrator lock on when " + reference + + log.log(Level.SEVERE, "Failed to get Orchestrator lock on when " + reference + eventDescription + ": " + e.getMessage()); return; } @@ -285,7 +285,7 @@ public class ZkStatusService implements StatusService { try (lock) { runnable.run(); } catch (RuntimeException e) { - log.log(LogLevel.ERROR, "Failed to clean up after " + reference + eventDescription + + log.log(Level.SEVERE, "Failed to clean up after " + reference + eventDescription + ": " + e.getMessage()); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java index 4b6a654a202..cda91994443 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java @@ -97,7 +97,7 @@ class CancellableImpl implements Cancellable, Runnable { runlet.close(); periodicExecutionCancellation.get().run(); } catch (Throwable e) { - logger.log(LogLevel.ERROR, "Failed run of periodic execution", e); + logger.log(Level.SEVERE, "Failed run of periodic execution", e); } } } -- cgit v1.2.3 From 729914d520026e931c9597e21831698aff30124c Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 15:59:28 +0200 Subject: LogLevel.FATAL -> Level.SEVERE --- .../clustercontroller/standalone/StandAloneClusterController.java | 6 +++--- .../java/com/yahoo/container/jdisc/component/Deconstructor.java | 2 +- .../main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java b/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java index 1e163cf7460..3242041b287 100644 --- a/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java +++ b/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java @@ -31,7 +31,7 @@ public class StandAloneClusterController { try{ app.stop(); } catch (Exception e) { - log.log(LogLevel.FATAL, "Failed to stop application '" + app.getName() + "': " + e.getMessage()); + log.log(Level.SEVERE, "Failed to stop application '" + app.getName() + "': " + e.getMessage()); e.printStackTrace(); return; } @@ -48,7 +48,7 @@ public class StandAloneClusterController { try{ myApp.start(); } catch (Exception e) { - log.log(LogLevel.FATAL, "Failed to start application '" + myApp.getName() + "': " + e.getMessage()); + log.log(Level.SEVERE, "Failed to start application '" + myApp.getName() + "': " + e.getMessage()); e.printStackTrace(); return; } @@ -56,7 +56,7 @@ public class StandAloneClusterController { try{ myApp.run(); } catch (Exception e) { - log.log(LogLevel.FATAL, "Application '" + myApp.getName() + "' runtime failure: " + e.getMessage()); + log.log(Level.SEVERE, "Application '" + myApp.getName() + "' runtime failure: " + e.getMessage()); e.printStackTrace(); } } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java index 1f111b8958c..d19362ecfb5 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java @@ -98,7 +98,7 @@ public class Deconstructor implements ComponentDeconstructor { } catch (Error e) { try { Duration shutdownDelay = getRandomizedShutdownDelay(); - log.log(LogLevel.FATAL, "Error when deconstructing component " + component + ". Will sleep for " + + log.log(Level.SEVERE, "Error when deconstructing component " + component + ". Will sleep for " + shutdownDelay.getSeconds() + " seconds then restart", e); Thread.sleep(shutdownDelay.toMillis()); } catch (InterruptedException exception) { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index 022c8254870..8b330c13dff 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -74,7 +74,7 @@ public class DocumentProcessingTask implements Runnable { } } catch (Error error) { try { - log.log(LogLevel.FATAL, Exceptions.toMessageString(error), error); + log.log(Level.SEVERE, Exceptions.toMessageString(error), error); } catch (Throwable t) { // do nothing } finally { -- cgit v1.2.3 From d64992c11dca04039a981b30c1bf023c9f5293ee Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 16:03:45 +0200 Subject: LogLevel.CONFIG -> Level.CONFIG --- .../java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java | 2 +- .../java/com/yahoo/container/jdisc/messagebus/SessionCache.java | 6 +++--- .../messagebus/protocol/DocumentRouteSelectorPolicy.java | 2 +- .../src/main/java/com/yahoo/logserver/handlers/HandlerThread.java | 2 +- .../src/main/java/com/yahoo/metrics/simple/MetricManager.java | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java index d0668772748..f3f9c914be8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -70,7 +70,7 @@ public class ConfigCurator { private ConfigCurator(Curator curator, int maxNodeSize) { this.curator = curator; this.maxNodeSize = maxNodeSize; - log.log(LogLevel.CONFIG, "Using jute max buffer size " + this.maxNodeSize); + log.log(Level.CONFIG, "Using jute max buffer size " + this.maxNodeSize); testZkConnection(); } diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index c8cef145a91..e62f6a8a21a 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -130,14 +130,14 @@ public final class SessionCache extends AbstractComponent { private static void logSystemInfo(ContainerMbusConfig containerMbusConfig, long maxPendingSize) { log.log(Level.FINE, "Running with maximum heap size of " + (Runtime.getRuntime().maxMemory() / 1024L / 1024L) + " MB"); - log.log(LogLevel.CONFIG, + log.log(Level.CONFIG, "Amount of memory reserved for container core: " + containerMbusConfig.containerCoreMemory() + " MB."); - log.log(LogLevel.CONFIG, + log.log(Level.CONFIG, "Running with document expansion factor " + containerMbusConfig.documentExpansionFactor() + ""); String msgLimit = (containerMbusConfig.maxpendingcount() == 0) ? "unlimited" : "" + containerMbusConfig.maxpendingcount(); - log.log(LogLevel.CONFIG, ("Starting message bus with max " + msgLimit + " pending messages and max " + + log.log(Level.CONFIG, ("Starting message bus with max " + msgLimit + " pending messages and max " + (((double) (maxPendingSize / 1024L)) / 1024.0d) + " pending megabytes.")); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java index ea8e87a8df0..8fbd1548f68 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java @@ -68,7 +68,7 @@ public class DocumentRouteSelectorPolicy DocumentSelector selector; try { selector = new DocumentSelector(route.selector()); - log.log(LogLevel.CONFIG, "Selector for route '" + route.name() + "' is '" + selector + "'."); + log.log(Level.CONFIG, "Selector for route '" + route.name() + "' is '" + selector + "'."); } catch (com.yahoo.document.select.parser.ParseException e) { error = "Error parsing selector '" + route.selector() + "' for route '" + route.name() + "; " + e.getMessage(); diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java index f1c08b9d390..057460323cd 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java @@ -79,7 +79,7 @@ public class HandlerThread extends Thread implements LogHandler { public HandlerThread(String name) { super(name); queue = new LinkedBlockingQueue<>(queueSize); - log.log(LogLevel.CONFIG, "logserver.queue.size=" + queueSize); + log.log(Level.CONFIG, "logserver.queue.size=" + queueSize); } /** diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java index 1b96178465a..a2a7c5b94ba 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java @@ -32,7 +32,7 @@ public class MetricManager extends AbstractComponent implements Provider updater) { - log.log(LogLevel.CONFIG, "setting up simple metrics gathering." + + log.log(Level.CONFIG, "setting up simple metrics gathering." + " reportPeriodSeconds=" + settings.reportPeriodSeconds() + ", pointsToKeepPerMetric=" + settings.pointsToKeepPerMetric()); metricsCollection = new ThreadLocalDirectory<>(updater); -- cgit v1.2.3 From c8837ab237c9631ab605eb9619c8ceea3036ad7e Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 18:29:36 +0200 Subject: LogLevel.FINE -> Level.FINE --- .../com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java | 2 +- .../java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java | 6 +++--- .../yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index 3e5c1f51990..58edc3a9afe 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -45,7 +45,7 @@ public class DeployHandlerLogger implements DeployLogger { entry.setString("level", level.getName()); entry.setString("message", fullMsg); // Also tee to a normal log, Vespa log for example, but use level fine - log.log(LogLevel.FINE, fullMsg); + log.log(Level.FINE, fullMsg); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 989fa10c202..e2cf84d6715 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -67,7 +67,7 @@ public class ZooKeeperClient { createZooKeeperNodes(); break; } catch (RuntimeException e) { - logger.log(LogLevel.FINE, "ZK init failed, retrying: " + e); + logger.log(Level.FINE, "ZK init failed, retrying: " + e); retries--; if (retries == 0) { throw e; @@ -262,7 +262,7 @@ public class ZooKeeperClient { writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); } - logger.log(LogLevel.FINE, configDefs.size() + " user config definitions"); + logger.log(Level.FINE, configDefs.size() + " user config definitions"); } private void writeConfigDefinition(String name, String namespace, String path, String data) { @@ -320,7 +320,7 @@ public class ZooKeeperClient { private void logFine(String msg) { if (logFine) { - logger.log(LogLevel.FINE, msg); + logger.log(Level.FINE, msg); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java index da450dfbedb..078836401ac 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java @@ -44,7 +44,7 @@ public class DeployHandlerLoggerTest { private void logMessages(DeployLogger logger) { logger.log(Level.FINE, "foobar"); logger.log(Level.FINEST, "foobar"); - logger.log(LogLevel.FINE, "baz"); + logger.log(Level.FINE, "baz"); logger.log(Level.WARNING, "baz"); } } -- cgit v1.2.3 From 91902bc7a20d8e6af77219e38fb645649610cd5e Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 16:28:41 +0200 Subject: Use LogLevel where appropriate in 'logserver' --- .../main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java | 2 +- .../src/main/java/com/yahoo/logserver/filter/LogFilterManager.java | 2 +- logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java | 2 +- .../java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java | 1 - .../com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java | 2 +- .../src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java | 2 +- .../ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java | 4 ++-- 7 files changed, 7 insertions(+), 8 deletions(-) diff --git a/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java index 39bdeaa624e..6e2a074cf8f 100644 --- a/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java +++ b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java @@ -8,7 +8,7 @@ import com.yahoo.log.LogMessage; import java.time.Instant; import java.util.List; -import java.util.logging.Level; +import com.yahoo.log.LogLevel; import static java.util.stream.Collectors.toList; diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java index 91f2ba3b96a..0e91bac164e 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/LogFilterManager.java @@ -4,7 +4,7 @@ package com.yahoo.logserver.filter; import java.util.HashMap; import java.util.Map; -import java.util.logging.Level; +import com.yahoo.log.LogLevel; /** * The LogFilterManager keeps track of associations between diff --git a/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java b/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java index fbd192690a3..144d498e290 100644 --- a/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java +++ b/logserver/src/main/java/com/yahoo/logserver/filter/MetricsFilter.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver.filter; -import java.util.logging.Level; +import com.yahoo.log.LogLevel; import com.yahoo.log.event.Count; import com.yahoo.log.event.CountGroup; import com.yahoo.log.event.Event; diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index 8ed6ec6de00..ee8c6e36cb8 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -14,7 +14,6 @@ import java.util.Calendar; import java.util.GregorianCalendar; import java.util.Iterator; import java.util.TimeZone; -import java.util.logging.Level; import java.util.logging.Logger; diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java index da19edb8413..9566c534d5d 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java @@ -12,7 +12,7 @@ import java.util.ArrayList; import java.util.logging.Logger; import java.util.logging.Level; -import java.util.logging.Level; +import com.yahoo.log.LogLevel; import com.yahoo.log.event.Event; import com.yahoo.logserver.filter.LevelFilter; import com.yahoo.log.LogMessage; diff --git a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java index d594df933f6..a398c2d6aff 100644 --- a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java +++ b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java @@ -5,7 +5,7 @@ import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; -import java.util.logging.Level; +import com.yahoo.log.LogLevel; import com.yahoo.log.event.MalformedEventException; import com.yahoo.log.InvalidLogFormatException; import com.yahoo.log.LogMessage; diff --git a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java index 982cb8a9de7..1dd79316eda 100644 --- a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java +++ b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java @@ -31,9 +31,9 @@ import static org.mockito.Mockito.verify; public class ArchiveLogMessagesMethodTest { private static final LogMessage MESSAGE_1 = - LogMessage.of(Instant.EPOCH.plus(1000, ChronoUnit.DAYS), "localhost", 12, 3456, "my-service", "my-component", LogLevel.ERROR, "My error message"); + LogMessage.of(Instant.EPOCH.plus(1000, ChronoUnit.DAYS), "localhost", 12, 3456, "my-service", "my-component", Level.SEVERE, "My error message"); private static final LogMessage MESSAGE_2 = - LogMessage.of(Instant.EPOCH.plus(5005, ChronoUnit.DAYS), "localhost", 12, 6543, "my-service", "my-component", LogLevel.INFO, "My info message"); + LogMessage.of(Instant.EPOCH.plus(5005, ChronoUnit.DAYS), "localhost", 12, 6543, "my-service", "my-component", Level.INFO, "My info message"); @Test public void server_dispatches_log_messages_from_log_request() { -- cgit v1.2.3 From f94cc116fffbf88ac70e603f91b4561ad743174f Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 19:14:32 +0200 Subject: LogLevel -> Level for isLoggable() --- .../clustercontroller/StateRestApiV2Handler.java | 2 +- .../clustercontroller/core/StateChangeHandler.java | 14 +++++----- .../clustercontroller/core/rpc/RpcServer.java | 2 +- .../clustercontroller/core/rpc/SlobrokClient.java | 2 +- .../model/producer/AbstractConfigProducer.java | 6 ++--- .../com/yahoo/vespa/config/proxy/MemoryCache.java | 2 +- .../config/server/application/Application.java | 2 +- .../config/server/rpc/DelayedConfigResponses.java | 4 +-- .../config/server/rpc/GetConfigProcessor.java | 2 +- .../yahoo/vespa/config/server/rpc/RpcServer.java | 8 +++--- .../config/server/tenant/TenantRequestHandler.java | 2 +- .../container/jdisc/LoggingRequestHandler.java | 2 +- .../jdisc/ThreadedHttpRequestHandler.java | 4 +-- .../com/yahoo/restapi/JacksonJsonResponse.java | 2 +- .../search/grouping/vespa/GroupingExecutor.java | 4 +-- .../search/searchers/InputCheckingSearcher.java | 2 +- .../yahoo/vespa/streamingvisitors/VdsVisitor.java | 18 ++++++------- .../java/com/yahoo/docproc/DocprocExecutor.java | 2 +- .../com/yahoo/docproc/SimpleDocumentProcessor.java | 8 +++--- .../jdisc/messagebus/MbusRequestContext.java | 2 +- .../docproc/jdisc/messagebus/MessageFactory.java | 2 +- .../docproc/util/SplitterDocumentProcessor.java | 6 ++--- .../java/com/yahoo/documentapi/ProgressToken.java | 12 ++++----- .../com/yahoo/documentapi/VisitorIterator.java | 30 +++++++++++----------- .../messagebus/MessageBusVisitorSession.java | 8 +++--- .../src/main/java/com/yahoo/logserver/Flusher.java | 2 +- .../java/com/yahoo/logserver/LogDispatcher.java | 2 +- .../yahoo/messagebus/DynamicThrottlePolicy.java | 4 +-- .../main/java/com/yahoo/messagebus/SendProxy.java | 6 ++--- .../yahoo/messagebus/network/rpc/RPCTarget.java | 4 +-- .../task/util/process/ProcessStarterImpl.java | 2 +- .../yahoo/vespa/http/server/ClientFeederV3.java | 4 +-- .../yahoo/feedapi/MessagePropertyProcessor.java | 2 +- .../main/java/com/yahoo/feedapi/SharedSender.java | 2 +- 34 files changed, 88 insertions(+), 88 deletions(-) diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java index ac5a7e3ed53..67aef138135 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java @@ -44,7 +44,7 @@ public class StateRestApiV2Handler extends JDiscHttpRequestHandler { } if (result.isEmpty()) { log.warning("Found no cluster controller in model config"); - } else if (log.isLoggable(LogLevel.DEBUG)) { + } else if (log.isLoggable(Level.FINE)) { StringBuilder sb = new StringBuilder(); sb.append("Found ").append(result.size()).append(" cluster controllers in model config:"); for (Map.Entry e : result.entrySet()) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index 85895b49efd..6bd6ab3c834 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -59,19 +59,19 @@ public class StateChangeHandler { final NodeState nodeState = currentState.getNodeState(node); if (nodeInfo != null && nodeState != null) { if (nodeState.getStartTimestamp() > nodeInfo.getStartTimestamp()) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, String.format("Storing away new start timestamp for node %s (%d)", node, nodeState.getStartTimestamp())); } nodeInfo.setStartTimestamp(nodeState.getStartTimestamp()); } if (nodeState.getStartTimestamp() > 0) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, String.format("Resetting timestamp in cluster state for node %s", node)); } ++startTimestampsReset; } - } else if (log.isLoggable(LogLevel.DEBUG)) { + } else if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, node + ": " + (nodeInfo == null ? "null" : nodeInfo.getStartTimestamp()) + ", " + (nodeState == null ? "null" : nodeState.getStartTimestamp())); @@ -152,7 +152,7 @@ public class StateChangeHandler { oldCount, newCount)); eventLog.add(NodeEvent.forBaseline(node, String.format("Altered min distribution bit count from %d to %d", oldCount, newCount), NodeEvent.Type.CURRENT, currentTime), isMaster); - } else if (log.isLoggable(LogLevel.DEBUG)) { + } else if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, String.format("Not altering state of %s in cluster state because new state is too similar: %s", node, currentState.getTextualDifference(reportedState))); } @@ -208,7 +208,7 @@ public class StateChangeHandler { } stateMayHaveChanged = true; - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, String.format("Got new wanted nodestate for %s: %s", node, currentState.getTextualDifference(proposedState))); } // Should be checked earlier before state was set in cluster @@ -415,7 +415,7 @@ public class StateChangeHandler { final NodeState reportedState, final NodeStateOrHostInfoChangeHandler nodeListener) { final long timeNow = timer.getCurrentTimeInMillis(); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, String.format("Finding new cluster state entry for %s switching state %s", node, currentState.getTextualDifference(reportedState))); } @@ -425,7 +425,7 @@ public class StateChangeHandler { } if (initializationProgressHasIncreased(currentState, reportedState)) { node.setInitProgressTime(timeNow); - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "Reset initialize timer on " + node + " to " + node.getInitProgressTime()); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index fb23e54b82a..bdce2b31036 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -306,7 +306,7 @@ public class RpcServer { } } } catch (Exception e) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); log.log(Level.FINE, "Failed RPC Request: " + sw); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index 3ef26b2a19f..c31fee36521 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -73,7 +73,7 @@ public class SlobrokClient implements NodeLookup { if (freshMirror) { freshMirror = false; } else if (cluster.getSlobrokGenerationCount() == mirrorVersion) { - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "Slobrok still at generation count " + cluster.getSlobrokGenerationCount() + ". Not updating."); } return false; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 4f811f90ed9..602e0c80d4b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -202,7 +202,7 @@ public abstract class AbstractConfigProducer responses = delayedConfigResponses.drainQueue(applicationId); String logPre = TenantRepository.logPre(applicationId); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, logPre + "Start of configReload: " + responses.size() + " requests on delayed requests queue"); } int responsesSent = 0; @@ -273,7 +273,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { // Doing cancel here deals with the case where the timer is already running or has not run, so // there is no need for any extra check. if (delayedConfigResponse.cancel()) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { logRequestDebug(LogLevel.DEBUG, logPre + "Timer cancelled for ", delayedConfigResponse.request); } // Do not wait for this request if we were unable to execute @@ -321,7 +321,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } public void respond(JRTServerConfigRequest request) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Trace at request return:\n" + request.getRequestTrace().toString()); } request.getRequest().returnRequest(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java index fbfbe11cdc6..1fb09b993fb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java @@ -80,7 +80,7 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host @Override public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional vespaVersion) { Application application = getApplication(appId, vespaVersion); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); } return application.resolveConfig(req, responseFactory); diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index d7b846cef5c..e0fc79b4be9 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -227,7 +227,7 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { public void failed(Throwable throwable) { long endTime = System.currentTimeMillis(); writeToLogs(endTime); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Got exception when writing to client: " + Exceptions.toMessageString(throwable)); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 164a27dbf54..9eec85f58b6 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -69,7 +69,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler @Override public final void handleRequest(Request request, BufferedContentChannel requestContent, ResponseHandler responseHandler) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "In " + this.getClass() + ".handleRequest()"); } com.yahoo.jdisc.http.HttpRequest jdiscRequest = asHttpRequest(request); @@ -191,7 +191,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler return responseHandler.handleResponse(httpResponse.getJdiscResponse()); } catch (Exception e) { metric.add(RENDERING_ERRORS, 1, null); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Error writing response to client - connection probably terminated " + "from client side.", e); } diff --git a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java index 699ebb7a705..94785819aa6 100644 --- a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java @@ -37,7 +37,7 @@ public class JacksonJsonResponse extends HttpResponse { @Override public void render(OutputStream outputStream) throws IOException { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { String json = jsonMapper.writeValueAsString(entity); log.log(Level.FINE, "Writing the following JSON to response output stream:\n" + json); outputStream.write(json.getBytes()); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index dbde0ad409f..d2d89e41879 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -225,7 +225,7 @@ public class GroupingExecutor extends Searcher { if (passList.isEmpty()) { throw new RuntimeException("No grouping request for pass " + pass + ", bug!"); } - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { for (Grouping grouping : passList) { log.log(Level.FINE, "Pass(" + pass + "), Grouping(" + grouping.getId() + "): " + grouping); } @@ -263,7 +263,7 @@ public class GroupingExecutor extends Searcher { ret = passResult; } } - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { for (Grouping grouping : groupingMap.values()) { log.log(Level.FINE, "Result Grouping(" + grouping.getId() + "): " + grouping); } diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java index 65b3fc5a805..3c0453f8900 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java @@ -51,7 +51,7 @@ public class InputCheckingSearcher extends Searcher { try { checkQuery(query); } catch (IllegalArgumentException e) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage()); } return new Result(query, ErrorMessage.createIllegalQuery(e.getMessage())); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index edfc12f3efa..2b0d8adc8ef 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -142,9 +142,9 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private int inferSessionTraceLevel(Query query) { int implicitLevel = traceLevelOverride; - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { implicitLevel = 9; - } else if (log.isLoggable(LogLevel.DEBUG)) { + } else if (log.isLoggable(Level.FINE)) { implicitLevel = 7; } return Math.max(query.getTraceLevel(), implicitLevel); @@ -342,7 +342,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } if (params.getControlHandler().getResult().code == VisitorControlHandler.CompletionCode.SUCCESS) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "VdsVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); } } else { @@ -384,7 +384,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } public void onSearchResult(SearchResult sr) { - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "Got SearchResult for query with selection " + params.getDocumentSelection()); } handleSearchResult(sr); @@ -393,7 +393,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private void handleSearchResult(SearchResult sr) { final int hitCountTotal = sr.getTotalHitCount(); final int hitCount = sr.getHitCount(); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Got SearchResult with " + hitCountTotal + " in total and " + hitCount + " hits in real for query with selection " + params.getDocumentSelection()); } @@ -412,14 +412,14 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } private void mergeGroupingMaps(Map newGroupingMap) { - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "mergeGroupingMaps: newGroupingMap = " + newGroupingMap); } for(Integer key : newGroupingMap.keySet()) { byte [] value = newGroupingMap.get(key); Grouping newGrouping = new Grouping(); - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "Received group with key " + key + " and size " + value.length); } BufferSerializer buf = new BufferSerializer( new GrowableByteBuffer(ByteBuffer.wrap(value)) ); @@ -440,7 +440,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } public void onDocumentSummary(DocumentSummary ds) { - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "Got DocumentSummary for query with selection " + params.getDocumentSelection()); } handleSummary(ds); @@ -448,7 +448,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private void handleSummary(DocumentSummary ds) { int summaryCount = ds.getSummaryCount(); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Got DocumentSummary with " + summaryCount + " summaries for query with selection " + params.getDocumentSelection()); } synchronized (summaryMap) { diff --git a/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java b/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java index ae636e12ba9..c80d9a166a8 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java @@ -119,7 +119,7 @@ public class DocprocExecutor { //might throw exception, which is OK: progress = call.call(processing); - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { logProgress(processing, progress, call); } diff --git a/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java index 5036ce59727..356df40af88 100644 --- a/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java @@ -34,7 +34,7 @@ public class SimpleDocumentProcessor extends DocumentProcessor { * @param put the DocumentPut to process. */ public void process(DocumentPut put) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Ignored " + put); } } @@ -47,7 +47,7 @@ public class SimpleDocumentProcessor extends DocumentProcessor { * @param update the DocumentUpdate to process. */ public void process(DocumentUpdate update) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Ignored " + update); } } @@ -60,7 +60,7 @@ public class SimpleDocumentProcessor extends DocumentProcessor { * @param remove the DocumentRemove to process. */ public void process(DocumentRemove remove) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Ignored " + remove); } } @@ -96,7 +96,7 @@ public class SimpleDocumentProcessor extends DocumentProcessor { process((DocumentRemove) op); } } catch (RuntimeException e) { - if (log.isLoggable(LogLevel.DEBUG) && initialSize != 1) { + if (log.isLoggable(Level.FINE) && initialSize != 1) { log.log(Level.FINE, "Processing of document failed, from processing.getDocumentOperations() containing " + initialSize + " DocumentOperation(s).", e); diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java index e8897f55a9e..2da494e19cf 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java @@ -91,7 +91,7 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { } } } - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Forwarding " + messages.size() + " messages from " + processings.size() + " processings."); } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java index ca7455e1305..92f4952b5b5 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java @@ -42,7 +42,7 @@ class MessageFactory { message.setTimeReceivedNow(); message.setTimeRemaining(requestMsg.getTimeRemainingNow()); message.getTrace().setLevel(requestMsg.getTrace().getLevel()); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Created '" + message.getClass().getName() + "', route = '" + message.getRoute() + "', priority = '" + message.getPriority().name() + diff --git a/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java index 7ec4ece3a3c..dfc36bfb641 100644 --- a/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java @@ -117,7 +117,7 @@ public class SplitterDocumentProcessor extends DocumentProcessor { static boolean doProcessOuterDocument(Object o, String documentTypeName) { if ( ! (o instanceof DocumentOperation)) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, o + " is not a DocumentOperation."); } return false; @@ -126,7 +126,7 @@ public class SplitterDocumentProcessor extends DocumentProcessor { DocumentOperation outerDocOp = (DocumentOperation)o; if ( ! (outerDocOp instanceof DocumentPut)) { //this is not a put, return - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Given DocumentOperation is not a DocumentPut, returning. (Was given " + outerDocOp + ")."); } @@ -137,7 +137,7 @@ public class SplitterDocumentProcessor extends DocumentProcessor { DocumentType type = outerDoc.getDataType(); if (!type.getName().equalsIgnoreCase(documentTypeName)) { //this is not the right document type - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Given Document is of wrong type, returning. (Was given " + outerDoc + ")."); } return false; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java index b7c78e28dbe..7e579a3ae6a 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java @@ -305,7 +305,7 @@ public class ProgressToken { && !progress.equals(FINISHED_BUCKET) && !superbucket.contains(progress) && !progress.contains(superbucket)) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "updateProgress called with non-contained bucket " + "pair " + superbucket + ":" + progress + ", but allowing anyway"); } @@ -322,7 +322,7 @@ public class ProgressToken { // If progress == Integer.MAX_VALUE, we're done. Otherwise, we're not if (!progress.equals(FINISHED_BUCKET)) { if (entry.getState() != BucketState.BUCKET_ACTIVE) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "updateProgress called with sub-bucket that was " + "not marked as active " + superbucket + ":" + progress); } @@ -361,13 +361,13 @@ public class ProgressToken { */ protected void addBucket(BucketId superbucket, BucketId progress, BucketState state) { if (progress.equals(FINISHED_BUCKET)) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Trying to add already finished superbucket " + superbucket + "; ignoring it"); } return; } - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "Adding bucket pair " + superbucket + ":" + progress + " with state " + state); } @@ -789,13 +789,13 @@ public class ProgressToken { // Must not merge if sibling isn't pending if (rightSibling != null) { assert(rightSibling.getState() == BucketState.BUCKET_PENDING); - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "Merging " + bucket + " with rhs " + rightCheck); } // If right sibling has progress, it will unfortunately have to // be discarded if (rightSibling.getProgress().getUsedBits() != 0 - && log.isLoggable(LogLevel.DEBUG)) { + && log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Bucket progress for " + rightCheck + " will be lost due to merging; potential for duplicates in result-set"); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java index 05324d957b8..0dd96275f9d 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java @@ -110,7 +110,7 @@ public class VisitorIterator { } if (!progress.isFinished()) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Importing unfinished progress token with " + "bits: " + progressToken.getDistributionBitCount() + ", active: " + progressToken.getActiveBucketCount() + @@ -122,7 +122,7 @@ public class VisitorIterator { if (!progress.isEmpty()) { // Lower all active to pending if (progressToken.getActiveBucketCount() > 0) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Progress token had active buckets upon range " + "construction. Setting these as pending"); } @@ -136,7 +136,7 @@ public class VisitorIterator { // Fixup for bucket cursor in case of bucket space downscaling correctTruncatedBucketCursor(); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Partial bucket space progress; continuing "+ "from position " + progressToken.getBucketCursor()); } @@ -195,7 +195,7 @@ public class VisitorIterator { // half a million splits to cover the same bucket space as that 1 // single-bit bucket once did if (isLosslessResetPossible()) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "At start of bucket space and all " + "buckets have no progress; doing a lossless reset " + "instead of splitting/merging"); @@ -245,7 +245,7 @@ public class VisitorIterator { BucketId rightCheck = new BucketId(lastMergedBucket.getUsedBits(), lastMergedBucket.getId() | (1L << (lastMergedBucket.getUsedBits() - 1))); if (pending.equals(rightCheck)) { - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "Skipped " + pending + ", as it was right sibling of " + lastMergedBucket); } @@ -262,7 +262,7 @@ public class VisitorIterator { } } } - if ((bucketsSplit > 0 || bucketsMerged > 0) && log.isLoggable(LogLevel.DEBUG)) { + if ((bucketsSplit > 0 || bucketsMerged > 0) && log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Existing progress' pending buckets had inconsistent " + "distribution bits; performed " + bucketsSplit + " split ops and " + bucketsMerged + " merge ops. Pending: " + pendingBefore + " -> " + @@ -285,7 +285,7 @@ public class VisitorIterator { progressToken.setBucketCursor(idx + 1); } } - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "New range bucket cursor is " + progressToken.getBucketCursor()); } @@ -334,7 +334,7 @@ public class VisitorIterator { // don't do anything at all yet with the set of pending if (progressToken.getActiveBucketCount() > 0) { flushActive = true; - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Holding off new/pending buckets and consistency " + "correction until all " + progress.getActiveBucketCount() + " active buckets have been updated"); @@ -349,7 +349,7 @@ public class VisitorIterator { // reset-checking to be performed correctInconsistentPending(distributionBitCount); if (delta > 0) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Increasing distribution bits for full bucket " + "space range source from " + progressToken.getDistributionBitCount() + " to " + distributionBitCount); @@ -359,7 +359,7 @@ public class VisitorIterator { // we go from eg. 3:0x02 to 4:0x02 to 5:02 etc. progressToken.setBucketCursor(progressToken.getBucketCursor() << delta); } else if (delta < 0) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Decreasing distribution bits for full bucket " + "space range source from " + progressToken.getDistributionBitCount() + " to " + distributionBitCount + " bits"); @@ -386,14 +386,14 @@ public class VisitorIterator { // We should now always flush active buckets before doing a // consistency fix. This simplifies things greatly assert(flushActive); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Received non-finished bucket " + superbucket + " with wrong distribution bit count (" + superbucket.getUsedBits() + "). Waiting to correct " + "until all active are done"); } } else { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Received finished bucket " + superbucket + " with wrong distribution bit count (" + superbucket.getUsedBits() + "). Waiting to correct " + @@ -404,7 +404,7 @@ public class VisitorIterator { if (progressToken.getActiveBucketCount() == 0) { if (flushActive) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "All active buckets flushed, " + "correcting progress token and continuing normal operation"); } @@ -508,7 +508,7 @@ public class VisitorIterator { // a no-op, since its buckets already are fixed at 32 used bits. progress.setDistributionBitCount(distributionBitCount); this.distributionBitCount = distributionBitCount; - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Set distribution bit count to " + distributionBitCount + " for explicit bucket source (no-op)"); } @@ -721,7 +721,7 @@ public class VisitorIterator { if (distributionBitCount != distBits) { bucketSource.setDistributionBitCount(distBits, progressToken); distributionBitCount = distBits; - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Set visitor iterator distribution bit count to " + distBits); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index 429c9bc0f51..1ad84cb0b25 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -574,7 +574,7 @@ public class MessageBusVisitorSession implements VisitorSession { 1, progressToken); } else { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "parameters specify explicit bucket set " + "to visit; using it rather than document selection (" + params.getBucketsToVisit().size() + " buckets given)"); @@ -754,7 +754,7 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public void run() { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Visitor session " + sessionName + ": Received message " + message); } try { @@ -809,7 +809,7 @@ public class MessageBusVisitorSession implements VisitorSession { Reply reply = msg.createReply(); msg.swapState(reply); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Visitor session " + sessionName + ": Received VisitorInfo with " + msg.getFinishedBuckets().size() + " finished buckets"); @@ -1101,7 +1101,7 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public void ack(AckToken token) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Visitor session " + sessionName + ": Sending ack " + token.ackObject); } diff --git a/logserver/src/main/java/com/yahoo/logserver/Flusher.java b/logserver/src/main/java/com/yahoo/logserver/Flusher.java index 21af3d77f34..07ecae66d90 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Flusher.java +++ b/logserver/src/main/java/com/yahoo/logserver/Flusher.java @@ -45,7 +45,7 @@ public class Flusher extends Thread { } else { h.flush(); } - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Flushing " + h); } } diff --git a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java index 7c34963ae4f..202d7ac966c 100644 --- a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java +++ b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java @@ -117,7 +117,7 @@ public class LogDispatcher implements LogHandler { } for (LogHandler h : handlers) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Flushing " + h.toString()); } h.flush(); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java index db464c1cd99..1281bd0b544 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java @@ -100,7 +100,7 @@ public class DynamicThrottlePolicy extends StaticThrottlePolicy { } else if (throughput > localMaxThroughput * 1.01) { localMaxThroughput = throughput; windowSize += weight*windowSizeIncrement; - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput); } } else { @@ -119,7 +119,7 @@ public class DynamicThrottlePolicy extends StaticThrottlePolicy { } else { windowSize += weight*windowSizeIncrement; } - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput + " efficiency " + efficiency); } } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java b/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java index d90cd2fb3da..d955a1a3d34 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java @@ -39,10 +39,10 @@ public class SendProxy implements MessageHandler, ReplyHandler { public void handleMessage(Message msg) { Trace trace = msg.getTrace(); if (trace.getLevel() == 0) { - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { trace.setLevel(9); logTrace = true; - } else if (log.isLoggable(LogLevel.DEBUG)) { + } else if (log.isLoggable(Level.FINE)) { trace.setLevel(6); logTrace = true; } @@ -60,7 +60,7 @@ public class SendProxy implements MessageHandler, ReplyHandler { if (logTrace) { if (reply.hasErrors()) { log.log(Level.FINE, "Trace for reply with error(s):\n" + reply.getTrace()); - } else if (log.isLoggable(LogLevel.SPAM)) { + } else if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "Trace for reply:\n" + reply.getTrace()); } Trace empty = new Trace(); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java index 25a5253d7be..2f81e44c74a 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java @@ -98,7 +98,7 @@ public class RPCTarget implements RequestWaiter { void resolveVersion(double timeout, VersionHandler handler) { boolean hasVersion = false; boolean shouldInvoke = false; - boolean shouldLog = log.isLoggable(LogLevel.DEBUG); + boolean shouldLog = log.isLoggable(Level.FINE); synchronized (this) { if (version != null) { if (shouldLog) { @@ -130,7 +130,7 @@ public class RPCTarget implements RequestWaiter { @Override public void handleRequestDone(Request req) { List handlers; - boolean shouldLog = log.isLoggable(LogLevel.DEBUG); + boolean shouldLog = log.isLoggable(Level.FINE); synchronized (this) { targetInvoked = false; if (req.checkReturnTypes("s")) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java index 3e2c14a2935..4fb7ff5571a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java @@ -16,7 +16,7 @@ public class ProcessStarterImpl implements ProcessStarter { @Override public ProcessApi2 start(ProcessBuilder processBuilder) { - if (logger.isLoggable(LogLevel.DEBUG)) { + if (logger.isLoggable(Level.FINE)) { logger.log(Level.FINE, "Spawning process: " + processBuilder.command()); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index 859b763ab9d..eccafc92c34 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -169,7 +169,7 @@ class ClientFeederV3 { try { operationId = streamReaderV3.getNextOperationId(requestInputStream); } catch (IOException ioe) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, Exceptions.toMessageString(ioe), ioe); } return Optional.empty(); @@ -182,7 +182,7 @@ class ClientFeederV3 { try { message = getNextMessage(operationId.get(), requestInputStream, settings); } catch (Exception e) { - if (log.isLoggable(LogLevel.WARNING)) { + if (log.isLoggable(Level.WARNING)) { log.log(Level.WARNING, Exceptions.toMessageString(e)); } metric.add(MetricNames.PARSE_ERROR, 1, null); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java index d5596d02804..0a070c9fe35 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java @@ -152,7 +152,7 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib defaultAbortOnDocumentError = feederOptions.abortOnDocumentError(); defaultAbortOnSendError = feederOptions.abortOnSendError(); - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { log.log(Level.FINE, "Received new config (" + "route: " + (defaultRoute != null ? defaultRoute : "") + ", timeout: " + defaultTimeoutMillis + " ms, retry enabled: " + retryEnabled + diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java index 6629ce5086a..d2e0434f273 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java @@ -124,7 +124,7 @@ public class SharedSender implements ReplyHandler { synchronized (metrics) { metrics.addReply(r); } - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, "Received reply for file " + owner.toString() + " count was " + owner.getPending().val()); } if (owner.isAborted()) { -- cgit v1.2.3 From 09b3ac7d50ba91f801a9d7cbd719a3cbf403932c Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 20:18:48 +0200 Subject: Use java.util.logging.Level in static imports. --- .../src/main/java/com/yahoo/vespa/model/HostSystem.java | 6 +++--- .../main/java/com/yahoo/container/di/ConfigRetriever.java | 14 +++++++------- .../src/main/java/com/yahoo/container/di/Container.java | 8 ++++---- .../container/di/componentgraph/core/ComponentNode.java | 6 +++--- .../com/yahoo/container/di/componentgraph/core/Node.java | 3 +-- .../container/di/componentgraph/cycle/CycleFinder.java | 8 ++++---- .../com/yahoo/container/jdisc/component/Deconstructor.java | 12 ++++++------ .../java/ai/vespa/metricsproxy/core/MetricsManager.java | 8 ++++---- .../metricsproxy/http/application/NodeMetricsClient.java | 6 +++--- .../java/ai/vespa/metricsproxy/metric/ExternalMetrics.java | 4 ++-- .../main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java | 4 ++-- .../src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java | 12 ++++++------ .../java/ai/vespa/metricsproxy/service/VespaServices.java | 10 +++++----- .../java/com/yahoo/vespa/hosted/testrunner/TestRunner.java | 4 ++-- 14 files changed, 52 insertions(+), 53 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 557a61ec211..292d4761d8a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -21,7 +21,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; /** * The parent node for all Host instances, and thus accessible @@ -120,7 +120,7 @@ public class HostSystem extends AbstractConfigProducer { HostResource host = getExistingHost(spec).orElseGet(() -> addNewHost(spec)); retAllocatedHosts.put(host, spec.membership().orElse(null)); } - retAllocatedHosts.keySet().forEach(host -> log.log(DEBUG, () -> "Allocated host " + host.getHostname() + " with flavor " + host.getFlavor())); + retAllocatedHosts.keySet().forEach(host -> log.log(FINE, () -> "Allocated host " + host.getHostname() + " with flavor " + host.getFlavor())); return retAllocatedHosts; } @@ -131,7 +131,7 @@ public class HostSystem extends AbstractConfigProducer { if (hosts.isEmpty()) { return Optional.empty(); } else { - log.log(DEBUG, () -> "Found existing host resource for " + key.hostname() + " with flavor " + hosts.get(0).getFlavor()); + log.log(FINE, () -> "Found existing host resource for " + key.hostname() + " with flavor " + hosts.get(0).getFlavor()); return Optional.of(hosts.get(0)); } } diff --git a/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java b/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java index fde4ecfd33a..6ed9c2a2994 100644 --- a/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java +++ b/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java @@ -16,7 +16,7 @@ import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; /** * @author Tony Vaagenes @@ -56,7 +56,7 @@ public final class ConfigRetriever { throw new IllegalArgumentException("Component config keys [" + componentConfigKeys + "] overlaps with bootstrap config keys [" + bootstrapKeys + "]"); - log.log(DEBUG, "getConfigs: " + componentConfigKeys); + log.log(FINE, "getConfigs: " + componentConfigKeys); Set> allKeys = new HashSet<>(componentConfigKeys); allKeys.addAll(bootstrapKeys); setupComponentSubscriber(allKeys); @@ -84,7 +84,7 @@ public final class ConfigRetriever { throw new IllegalArgumentException( "Component config keys [" + componentConfigKeys + "] overlaps with bootstrap config keys [" + bootstrapKeys + "]"); } - log.log(DEBUG, "getConfigsOnce: " + componentConfigKeys); + log.log(FINE, "getConfigsOnce: " + componentConfigKeys); Set> allKeys = new HashSet<>(componentConfigKeys); allKeys.addAll(bootstrapKeys); @@ -97,7 +97,7 @@ public final class ConfigRetriever { private Optional getConfigsOptional(long leastGeneration, boolean restartOnRedeploy) { long newestComponentGeneration = componentSubscriber.waitNextGeneration(); - log.log(DEBUG, "getConfigsOptional: new component generation: " + newestComponentGeneration); + log.log(FINE, "getConfigsOptional: new component generation: " + newestComponentGeneration); // leastGeneration is only used to ensure newer generation when the previous generation was invalidated due to an exception if (newestComponentGeneration < leastGeneration) { @@ -106,13 +106,13 @@ public final class ConfigRetriever { return Optional.empty(); } else if (bootstrapSubscriber.generation() < newestComponentGeneration) { long newestBootstrapGeneration = bootstrapSubscriber.waitNextGeneration(); - log.log(DEBUG, "getConfigsOptional: new bootstrap generation: " + bootstrapSubscriber.generation()); + log.log(FINE, "getConfigsOptional: new bootstrap generation: " + bootstrapSubscriber.generation()); Optional bootstrapConfig = bootstrapConfigIfChanged(); if (bootstrapConfig.isPresent()) { return bootstrapConfig; } else { if (newestBootstrapGeneration == newestComponentGeneration) { - log.log(DEBUG, "Got new components configs with unchanged bootstrap configs."); + log.log(FINE, "Got new components configs with unchanged bootstrap configs."); return componentsConfigIfChanged(); } else { // This should not be a normal case, and hence a warning to allow investigation. @@ -155,7 +155,7 @@ public final class ConfigRetriever { componentSubscriber.close(); componentSubscriberKeys = keys; try { - log.log(DEBUG, "Setting up new component subscriber for keys: " + keys); + log.log(FINE, "Setting up new component subscriber for keys: " + keys); componentSubscriber = subscribe.apply(keys); } catch (Throwable e) { log.log(Level.WARNING, "Failed setting up subscriptions for component configs: " + e.getMessage()); diff --git a/container-di/src/main/java/com/yahoo/container/di/Container.java b/container-di/src/main/java/com/yahoo/container/di/Container.java index 9a9245f4ba2..d8fec35633b 100644 --- a/container-di/src/main/java/com/yahoo/container/di/Container.java +++ b/container-di/src/main/java/com/yahoo/container/di/Container.java @@ -30,7 +30,7 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; /** * @author gjoranv @@ -139,7 +139,7 @@ public class Container { while (true) { snapshot = configurer.getConfigs(graph.configKeys(), leastGeneration, restartOnRedeploy); - log.log(DEBUG, String.format("createNewGraph:\n" + "graph.configKeys = %s\n" + "graph.generation = %s\n" + "snapshot = %s\n", + log.log(FINE, String.format("createNewGraph:\n" + "graph.configKeys = %s\n" + "graph.generation = %s\n" + "snapshot = %s\n", graph.configKeys(), graph.generation(), snapshot)); if (snapshot instanceof BootstrapConfigs) { @@ -148,7 +148,7 @@ public class Container { "Got bootstrap configs out of sequence for old config generation %d.\n" + "Previous config generation is %d", getBootstrapGeneration(), previousConfigGeneration)); } - log.log(DEBUG, + log.log(FINE, String.format( "Got new bootstrap generation\n" + "bootstrap generation = %d\n" + "components generation: %d\n" + "previous generation: %d\n", @@ -165,7 +165,7 @@ public class Container { break; } } - log.log(DEBUG, + log.log(FINE, String.format( "Got components configs,\n" + "bootstrap generation = %d\n" + "components generation: %d\n" + "previous generation: %d", diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java index 6662ba85e5d..92695565d47 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java @@ -29,7 +29,7 @@ import java.util.stream.Collectors; import static com.yahoo.container.di.componentgraph.core.Exceptions.cutStackTraceAtConstructor; import static com.yahoo.container.di.componentgraph.core.Exceptions.removeStackTrace; import static com.yahoo.container.di.componentgraph.core.Keys.createKey; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; /** * @author Tony Vaagenes @@ -148,9 +148,9 @@ public class ComponentNode extends Node { Object instance; try { - log.log(DEBUG, "Constructing " + idAndType()); + log.log(FINE, "Constructing " + idAndType()); instance = constructor.newInstance(actualArguments.toArray()); - log.log(DEBUG, "Finished constructing " + idAndType()); + log.log(FINE, "Finished constructing " + idAndType()); } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { StackTraceElement dependencyInjectorMarker = new StackTraceElement("============= Dependency Injection =============", "newInstance", null, -1); diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java index 9ed89a6091a..3afc8bb817c 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java @@ -15,8 +15,7 @@ import java.util.Optional; import java.util.Set; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; -import static com.yahoo.log.LogLevel.SPAM; +import static java.util.logging.Level.FINE; /** * @author Tony Vaagenes diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java index ed9b1350b1c..327949bb8d0 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java @@ -11,7 +11,7 @@ import java.util.stream.Collectors; import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.BLACK; import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.GRAY; import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.WHITE; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.Collections.singletonList; @@ -58,7 +58,7 @@ public class CycleFinder { if (visitDepthFirst(vertex, new ArrayList<>(singletonList(vertex)))) { if (cycle == null) throw new IllegalStateException("Null cycle - this should never happen"); if (cycle.isEmpty()) throw new IllegalStateException("Empty cycle - this should never happen"); - log.log(DEBUG, "Cycle detected: " + cycle); + log.log(FINE, "Cycle detected: " + cycle); return cycle; } } @@ -68,7 +68,7 @@ public class CycleFinder { private boolean visitDepthFirst(T vertex, List path) { colors.put(vertex, GRAY); - log.log(DEBUG, "Vertex start " + vertex + " - colors: " + colors + " - path: " + path); + log.log(FINE, "Vertex start " + vertex + " - colors: " + colors + " - path: " + path); for (T adjacent : graph.getAdjacent(vertex)) { path.add(adjacent); if (colors.get(adjacent) == GRAY) { @@ -81,7 +81,7 @@ public class CycleFinder { path.remove(adjacent); } colors.put(vertex, BLACK); - log.log(DEBUG, "Vertex end " + vertex + " - colors: " + colors + " - path: " + path); + log.log(FINE, "Vertex end " + vertex + " - colors: " + colors + " - path: " + path); return false; } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java index d19362ecfb5..e60e8d407cd 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java @@ -19,7 +19,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; import static java.util.logging.Level.SEVERE; import static java.util.logging.Level.WARNING; @@ -52,11 +52,11 @@ public class Deconstructor implements ComponentDeconstructor { } } else if (component instanceof Provider) { // TODO Providers should most likely be deconstructed similarly to AbstractComponent - log.log(DEBUG, () -> "Starting deconstruction of provider " + component); + log.log(FINE, () -> "Starting deconstruction of provider " + component); ((Provider) component).deconstruct(); - log.log(DEBUG, () -> "Finished deconstruction of provider " + component); + log.log(FINE, () -> "Finished deconstruction of provider " + component); } else if (component instanceof SharedResource) { - log.log(DEBUG, () -> "Releasing container reference to resource " + component); + log.log(FINE, () -> "Releasing container reference to resource " + component); // No need to delay release, as jdisc does ref-counting ((SharedResource) component).release(); } @@ -89,10 +89,10 @@ public class Deconstructor implements ComponentDeconstructor { @Override public void run() { for (var component : components) { - log.log(DEBUG, () -> "Starting deconstruction of component " + component); + log.log(FINE, () -> "Starting deconstruction of component " + component); try { component.deconstruct(); - log.log(DEBUG, () -> "Finished deconstructing of component " + component); + log.log(FINE, () -> "Finished deconstructing of component " + component); } catch (Exception | NoClassDefFoundError e) { // May get class not found due to it being already unloaded log.log(WARNING, "Exception thrown when deconstructing component " + component, e); } catch (Error e) { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index 53a05ef88f0..33827634ebf 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -23,7 +23,7 @@ import java.util.stream.Collectors; import static ai.vespa.metricsproxy.metric.ExternalMetrics.extractConfigserverDimensions; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.stream.Collectors.toList; /** @@ -91,17 +91,17 @@ public class MetricsManager { public List getMetricsAsBuilders(List services, Instant startTime) { if (services.isEmpty()) return Collections.emptyList(); - log.log(DEBUG, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); + log.log(FINE, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); vespaServices.updateServices(services); List result = vespaMetrics.getMetrics(services); - log.log(DEBUG, () -> "Got " + result.size() + " metrics packets for vespa services."); + log.log(FINE, () -> "Got " + result.size() + " metrics packets for vespa services."); purgeStaleMetrics(); List externalPackets = externalMetrics.getMetrics().stream() .filter(MetricsPacket.Builder::hasMetrics) .collect(toList()); - log.log(DEBUG, () -> "Got " + externalPackets.size() + " external metrics packets with whitelisted metrics."); + log.log(FINE, () -> "Got " + externalPackets.size() + " external metrics packets with whitelisted metrics."); result.addAll(externalPackets); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java index 145eef3f745..a93b5eb31c0 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.Collections.emptyList; /** @@ -65,13 +65,13 @@ public class NodeMetricsClient { private Snapshot retrieveMetrics(ConsumerId consumer) { String metricsUri = node.metricsUri(consumer).toString(); - log.log(DEBUG, () -> "Retrieving metrics from host " + metricsUri); + log.log(FINE, () -> "Retrieving metrics from host " + metricsUri); try { String metricsJson = httpClient.execute(new HttpGet(metricsUri), new BasicResponseHandler()); var newMetrics = GenericJsonUtil.toMetricsPackets(metricsJson); snapshotsRetrieved ++; - log.log(DEBUG, () -> "Successfully retrieved " + newMetrics.size() + " metrics packets from " + metricsUri); + log.log(FINE, () -> "Successfully retrieved " + newMetrics.size() + " metrics packets from " + metricsUri); return new Snapshot(Instant.now(clock), newMetrics); } catch (IOException e) { log.warning("Unable to retrieve metrics from " + metricsUri + ": " + Exceptions.toMessageString(e)); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java index 0a2dbc9a910..337967f3075 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java @@ -21,7 +21,7 @@ import java.util.logging.Logger; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.stream.Collectors.toCollection; /** @@ -53,7 +53,7 @@ public class ExternalMetrics { public void setExtraMetrics(List externalPackets) { // TODO: Metrics filtering per consumer is not yet implemented. // Split each packet per metric, and re-aggregate based on the metrics each consumer wants. Then filter out all packages with no consumers. - log.log(DEBUG, () -> "Setting new external metrics with " + externalPackets.size() + " metrics packets."); + log.log(FINE, () -> "Setting new external metrics with " + externalPackets.size() + " metrics packets."); externalPackets.forEach(packet -> { packet.addConsumers(consumers.getAllConsumers()) .retainMetrics(metricsToRetain()) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java index 269b70c8316..c09dc44d3fd 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java @@ -11,7 +11,7 @@ import com.yahoo.jrt.Transport; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; /** @@ -30,7 +30,7 @@ public class RpcConnector extends AbstractComponent { Spec spec = new Spec(config.port()); try { acceptor = supervisor.listen(spec); - log.log(DEBUG, "Listening on " + spec.host() + ":" + acceptor.port()); + log.log(FINE, "Listening on " + spec.host() + ":" + acceptor.port()); } catch (ListenFailedException e) { stop(); log.log(INFO, "Failed listening at " + spec.host() + ":" + spec.port()); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java index ba84347c7ea..e634d533e5c 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java @@ -20,7 +20,7 @@ import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; import static ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil.toMetricsPackets; import static ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil.toYamasArray; import static com.yahoo.collections.CollectionUtil.mkString; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; @@ -46,7 +46,7 @@ public class RpcServer { this.vespaServices = vespaServices; this.metricsManager = metricsManager; addMethods(connector); - log.log(DEBUG, "RPC server created"); + log.log(FINE, "RPC server created"); } private void addMethods(RpcConnector connector) { @@ -120,13 +120,13 @@ public class RpcServer { Instant startTime = Instant.now(); req.detach(); String service = req.parameters().get(0).asString(); - log.log(DEBUG, () -> "getMetricsForYamas called at " + startTime + " with argument: " + service); + log.log(FINE, () -> "getMetricsForYamas called at " + startTime + " with argument: " + service); List services = vespaServices.getMonitoringServices(service); - log.log(DEBUG, () -> "Getting metrics for services: " + mkString(services, "[", ", ", "]")); + log.log(FINE, () -> "Getting metrics for services: " + mkString(services, "[", ", ", "]")); if (services.isEmpty()) setNoServiceError(req, service); else withExceptionHandling(req, () -> { List packets = metricsManager.getMetrics(services, startTime); - log.log(DEBUG,() -> "Returning metrics packets:\n" + mkString(packets, "\n")); + log.log(FINE,() -> "Returning metrics packets:\n" + mkString(packets, "\n")); req.returnValues().add(new StringValue(toYamasArray(packets).serialize())); }); req.returnRequest(); @@ -146,7 +146,7 @@ public class RpcServer { void setExtraMetrics(Request req) { String metricsJson = req.parameters().get(0).asString(); - log.log(DEBUG, "setExtraMetrics called with argument: " + metricsJson); + log.log(FINE, "setExtraMetrics called with argument: " + metricsJson); withExceptionHandling(req, () -> metricsManager.setExtraMetrics(toMetricsPackets(metricsJson))); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java index 76a4ae5046b..e01a68a1f7f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java @@ -15,7 +15,7 @@ import java.util.logging.Logger; import static ai.vespa.metricsproxy.core.MetricsConsumers.toUnmodifiableLinkedMap; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; /** * Creates representations for the Vespa services running on the node, @@ -47,12 +47,12 @@ public class VespaServices { private List createServices(VespaServicesConfig servicesConfig, String monitoringSystemName) { List services = new ArrayList<>(); for (Service s : servicesConfig.service()) { - log.log(DEBUG, "Creating service " + s.name()); + log.log(FINE, "Creating service " + s.name()); VespaService vespaService = VespaService.create(s.name(), s.configId(), s.port(), monitoringSystemName, createServiceDimensions(s)); services.add(vespaService); } - log.log(DEBUG, "Created new services: " + services.size()); + log.log(FINE, "Created new services: " + services.size()); return services; } @@ -62,7 +62,7 @@ public class VespaServices { */ public final void updateServices(List services) { if (sentinel != null) { - log.log(DEBUG, "Updating services "); + log.log(FINE, "Updating services "); sentinel.updateServiceStatuses(services); } } @@ -103,7 +103,7 @@ public class VespaServices { List myServices = new ArrayList<>(); for (VespaService s : services) { - log.log(DEBUG, () -> "getMonitoringServices. service=" + service + ", checking against " + s + ", which has monitoring name " + s.getMonitoringName()); + log.log(FINE, () -> "getMonitoringServices. service=" + service + ", checking against " + s + ", which has monitoring name " + s.getMonitoringName()); if (s.getMonitoringName().equalsIgnoreCase(service)) { myServices.add(s); } diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java index bf95b2ef6c5..7a94bbc1e47 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java @@ -29,9 +29,9 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.log.LogLevel.ERROR; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.logging.Level.INFO; +import static java.util.logging.Level.SEVERE; /** * @author valerijf @@ -167,7 +167,7 @@ public class TestRunner { success = mavenProcess.waitFor() == 0; } catch (Exception exception) { - LogRecord record = new LogRecord(ERROR, "Failed to execute maven command: " + String.join(" ", builder.command())); + LogRecord record = new LogRecord(SEVERE, "Failed to execute maven command: " + String.join(" ", builder.command())); record.setThrown(exception); logger.log(record); log.put(record.getSequenceNumber(), record); -- cgit v1.2.3 From d588f01d40105a13129b8e007a7f65231a6e1ca5 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 20:51:57 +0200 Subject: Map remaining DEBUG/SPAM/ERROR/FATAL -> Level.FINE/FINEST/SEVERE --- .../com/yahoo/vespa/clustercontroller/core/EventLog.java | 4 ++-- .../vespa/clustercontroller/core/StateChangeHandler.java | 4 ++-- .../clustercontroller/core/SystemStateBroadcaster.java | 2 +- .../core/status/statuspage/StatusPageServer.java | 6 +++--- .../vespa/clustercontroller/core/StateGatherTest.java | 2 +- .../utils/staterestapi/server/RestApiHandler.java | 4 ++-- .../vespa/config/server/deploy/DeployHandlerLogger.java | 4 ++-- .../vespa/config/server/rpc/GetConfigProcessor.java | 2 +- .../com/yahoo/vespa/config/server/rpc/RpcServer.java | 2 +- .../config/server/session/LocalSessionStateWatcher.java | 2 +- .../container/jdisc/ContentChannelOutputStream.java | 4 ++-- .../rendering/AsynchronousSectionedRenderer.java | 2 +- .../java/com/yahoo/container/jdisc/LoggingTestCase.java | 4 ++-- .../vespa/streamingvisitors/VdsStreamingSearcher.java | 4 ++-- .../hosted/controller/api/integration/LogEntry.java | 2 +- .../hosted/controller/deployment/DeploymentTester.java | 6 +++--- .../vespa/filedistribution/FileReferenceDownloader.java | 2 +- .../vespa/logserver/protocol/ProtobufSerialization.java | 16 ++++++++-------- .../java/com/yahoo/messagebus/SendProxyTestCase.java | 4 ++-- .../hosted/node/admin/maintenance/acl/AclMaintainer.java | 4 ++-- .../identity/AthenzCredentialsMaintainer.java | 10 +++++----- .../vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java | 6 +++--- .../provisioning/NodeRepositoryProvisioner.java | 2 +- .../yahoo/vespa/hosted/testrunner/TestRunnerHandler.java | 2 +- .../java/com/yahoo/vespa/http/server/ClientFeederV3.java | 4 ++-- 25 files changed, 52 insertions(+), 52 deletions(-) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java index e0f3b294c4d..4af4b2b15a1 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java @@ -57,9 +57,9 @@ public class EventLog implements EventLogInterface { } if (e instanceof NodeEvent) { - addNodeOnlyEvent((NodeEvent)e, logInfo ? LogLevel.INFO: LogLevel.DEBUG); + addNodeOnlyEvent((NodeEvent)e, logInfo ? LogLevel.INFO: Level.FINE); } else { - log.log(logInfo ? LogLevel.INFO : LogLevel.DEBUG, e.toString()); + log.log(logInfo ? LogLevel.INFO : Level.FINE, e.toString()); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index 6bd6ab3c834..abdb6255ca2 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -118,7 +118,7 @@ public class StateChangeHandler { final NodeStateOrHostInfoChangeHandler nodeListener) { final NodeState currentState = currentClusterState.getNodeState(node.getNode()); - final LogLevel level = (currentState.equals(reportedState) && node.getVersion() == 0) ? LogLevel.SPAM : LogLevel.DEBUG; + final LogLevel level = (currentState.equals(reportedState) && node.getVersion() == 0) ? Level.FINEST : Level.FINE; if (log.isLoggable(level)) { log.log(level, String.format("Got nodestate reply from %s: %s (Current state is %s)", node, node.getReportedState().getTextualDifference(reportedState), currentState.toString(true))); @@ -134,7 +134,7 @@ public class StateChangeHandler { if (reportedState.getState().equals(State.DOWN)) { eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Failed to get node state: " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); } else { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Now reporting state " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), LogLevel.DEBUG); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Now reporting state " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), Level.FINE); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java index 645ad694a0c..95e03d1f6f1 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java @@ -66,7 +66,7 @@ public class SystemStateBroadcaster { long time = timer.getCurrentTimeInMillis(); Long lastReported = lastErrorReported.get(info.getNode()); boolean alreadySeen = (lastReported != null && time - lastReported < minTimeBetweenNodeErrorLogging); - log.log((nodeOk && !alreadySeen) ? LogLevel.WARNING : LogLevel.DEBUG, message); + log.log((nodeOk && !alreadySeen) ? LogLevel.WARNING : Level.FINE, message); if (!alreadySeen) { lastErrorReported.put(info.getNode(), time); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index 55241d8310d..3028ffe6e2c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -146,7 +146,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { } catch (SocketTimeoutException e) { // Ignore, since timeout is set to 100 ms } catch (java.io.IOException e) { - log.log(shouldBeConnected ? LogLevel.WARNING : LogLevel.DEBUG, "Caught IO exception in ServerSocket.accept(): " + e.getMessage()); + log.log(shouldBeConnected ? LogLevel.WARNING : Level.FINE, "Caught IO exception in ServerSocket.accept(): " + e.getMessage()); } if (connection == null) continue; log.log(Level.FINE, "Got a status page request."); @@ -235,7 +235,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { output.write(header.toString().getBytes()); output.write(response.getOutputStream().toByteArray()); } catch (java.io.IOException e) { - log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? LogLevel.DEBUG : LogLevel.INFO, + log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? Level.FINE : LogLevel.INFO, "Failed to process HTTP request : " + e.getMessage()); } catch (Exception e) { log.log(Level.WARNING, "Caught exception in HTTP server thread: " @@ -244,7 +244,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { if (output != null) try { output.close(); } catch (IOException e) { - log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? LogLevel.DEBUG : LogLevel.INFO, + log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? Level.FINE : LogLevel.INFO, "Failed to close output stream on socket " + connection + ": " + e.getMessage()); } if (connection != null) try{ diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java index 695e2954546..7e2d8d09e1b 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java @@ -25,7 +25,7 @@ public class StateGatherTest extends FleetControllerTest { @Test public void testAlwaysHavePendingGetNodeStateRequestTowardsNodes() throws Exception { - Logger.getLogger(NodeStateGatherer.class.getName()).setLevel(LogLevel.SPAM); + Logger.getLogger(NodeStateGatherer.class.getName()).setLevel(Level.FINEST); startingTest("StateGatherTest::testOverlappingGetNodeStateRequests"); FleetControllerOptions options = defaultOptions("mycluster"); options.nodeStateRequestTimeoutMS = 10 * 60 * 1000; diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java index fc7b3a96d53..953f1000236 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java @@ -103,7 +103,7 @@ public class RestApiHandler implements HttpRequestHandler { return new JsonHttpResult().setJson(jsonWriter.createJson(setResponse)); } } catch (OtherMasterException exception) { - logRequestException(request, exception, LogLevel.DEBUG); + logRequestException(request, exception, Level.FINE); JsonHttpResult result = new JsonHttpResult(); result.setHttpCode(307, "Temporary Redirect"); result.addHeader("Location", getMasterLocationUrl(request, exception.getHost(), exception.getPort())); @@ -130,7 +130,7 @@ public class RestApiHandler implements HttpRequestHandler { result.setJson(jsonWriter.createErrorJson(exception.getMessage())); return result; } catch (Exception exception) { - logRequestException(request, exception, LogLevel.ERROR); + logRequestException(request, exception, Level.SEVERE); JsonHttpResult result = new JsonHttpResult(); result.setHttpCode(500, "Failed to process request"); result.setJson(jsonWriter.createErrorJson(exception.getClass().getName() + ": " + exception.getMessage())); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index 58edc3a9afe..f45e2cf6285 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -34,8 +34,8 @@ public class DeployHandlerLogger implements DeployLogger { @Override public void log(Level level, String message) { if ((level == LogLevel.FINE || - level == LogLevel.DEBUG || - level == LogLevel.SPAM) && + level == Level.FINE || + level == Level.FINEST) && !verbose) { return; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 8c46ec32adb..cf6890e22d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -49,7 +49,7 @@ class GetConfigProcessor implements Runnable { private void respond(JRTServerConfigRequest request) { Request req = request.getRequest(); if (req.isError()) { - Level logLevel = (req.errorCode() == ErrorCode.APPLICATION_NOT_LOADED) ? LogLevel.DEBUG : LogLevel.INFO; + Level logLevel = (req.errorCode() == ErrorCode.APPLICATION_NOT_LOADED) ? Level.FINE : LogLevel.INFO; log.log(logLevel, logPre + req.errorMessage()); } rpcServer.respond(request); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 3bc67e59fd9..8082b340970 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -274,7 +274,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { // there is no need for any extra check. if (delayedConfigResponse.cancel()) { if (log.isLoggable(Level.FINE)) { - logRequestDebug(LogLevel.DEBUG, logPre + "Timer cancelled for ", delayedConfigResponse.request); + logRequestDebug(Level.FINE, logPre + "Timer cancelled for ", delayedConfigResponse.request); } // Do not wait for this request if we were unable to execute if (addToRequestQueue(delayedConfigResponse.request, false, completionService)) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java index 268a755b266..eb5770972ff 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java @@ -37,7 +37,7 @@ public class LocalSessionStateWatcher { // Will delete session if it exists in local session repo private void sessionChanged(Session.Status status) { long sessionId = session.getSessionId(); - log.log(status == Session.Status.DELETE ? LogLevel.INFO : LogLevel.DEBUG, + log.log(status == Session.Status.DELETE ? LogLevel.INFO : Level.FINE, session.logPre() + "Session change: Local session " + sessionId + " changed status to " + status); if (status.equals(Session.Status.DELETE) && localSessionRepo.getSession(sessionId) != null) { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java b/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java index dde4e8b73e4..9c48955bf4c 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java @@ -159,9 +159,9 @@ public class ContentChannelOutputStream extends OutputStream implements Writable Level logLevel; synchronized (failLock) { if (failed) { - logLevel = LogLevel.SPAM; + logLevel = Level.FINEST; } else { - logLevel = LogLevel.DEBUG; + logLevel = Level.FINE; } failed = true; } diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java index cedd5b97ed6..6c5ac7718d5 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java @@ -529,7 +529,7 @@ public abstract class AsynchronousSectionedRenderer e } catch (Exception e) { Level level = LogLevel.WARNING; if ((e instanceof IOException)) { - level = LogLevel.DEBUG; + level = Level.FINE; if ( ! clientClosed) { clientClosed = true; onClientClosed(); diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java index dce791d375f..876fd0d2b57 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java @@ -102,8 +102,8 @@ public class LoggingTestCase { stream.send(createData()); stream.flush(); assertNull(logChecker.errorCounter.get(LogLevel.INFO)); - assertEquals(1, logChecker.errorCounter.get(LogLevel.DEBUG).intValue()); - assertEquals(2, logChecker.errorCounter.get(LogLevel.SPAM).intValue()); + assertEquals(1, logChecker.errorCounter.get(Level.FINE).intValue()); + assertEquals(2, logChecker.errorCounter.get(Level.FINEST).intValue()); } } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 9ac1cb33d1b..56b5a826cba 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -300,9 +300,9 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { String expUserId = query.properties().getString(streamingUserid); String expGroupName = query.properties().getString(streamingGroupname); - LogLevel logLevel = LogLevel.ERROR; + LogLevel logLevel = Level.SEVERE; if (skippedEarlierResult) { - logLevel = LogLevel.DEBUG; + logLevel = Level.FINE; } DocumentId docId; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java index b0170486eca..c16e72e3ad1 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java @@ -105,7 +105,7 @@ public class LogEntry { public static Type typeOf(Level level) { return level.intValue() < LogLevel.INFO.intValue() ? Type.debug : level.intValue() < LogLevel.WARNING.intValue() ? Type.info - : level.intValue() < LogLevel.ERROR.intValue() ? Type.warning + : level.intValue() < Level.SEVERE.intValue() ? Type.warning : Type.error; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 926f5a352cb..0dccea41ae1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -85,9 +85,9 @@ public class DeploymentTester { outstandingChangeDeployer = new OutstandingChangeDeployer(tester.controller(), maintenanceInterval, jobControl); // Get deployment job logs to stderr. - Logger.getLogger("").setLevel(LogLevel.DEBUG); - Logger.getLogger(InternalStepRunner.class.getName()).setLevel(LogLevel.DEBUG); - tester.configureDefaultLogHandler(handler -> handler.setLevel(LogLevel.DEBUG)); + Logger.getLogger("").setLevel(Level.FINE); + Logger.getLogger(InternalStepRunner.class.getName()).setLevel(Level.FINE); + tester.configureDefaultLogHandler(handler -> handler.setLevel(Level.FINE)); // Mock Athenz domain to allow launch of service AthenzDbMock.Domain domain = tester.athenzDb().getOrCreateDomain(new com.yahoo.vespa.athenz.api.AthenzDomain(ATHENZ_DOMAIN)); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 11dd5195ee0..78afcd72056 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -117,7 +117,7 @@ public class FileReferenceDownloader { request.parameters().add(new Int32Value(fileReferenceDownload.downloadFromOtherSourceIfNotFound() ? 0 : 1)); connection.invokeSync(request, (double) rpcTimeout.getSeconds()); - Level logLevel = (retryCount > 0 ? LogLevel.INFO : LogLevel.DEBUG); + Level logLevel = (retryCount > 0 ? LogLevel.INFO : Level.FINE); if (validateResponse(request)) { log.log(logLevel, () -> "Request callback, OK. Req: " + request + "\nSpec: " + connection + ", retry count " + retryCount); if (request.returnValues().get(0).asInt32() == 0) { diff --git a/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java index 6e2a074cf8f..3f040821a7f 100644 --- a/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java +++ b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java @@ -82,9 +82,9 @@ class ProtobufSerialization { private static Level fromLogMessageLevel(LogProtocol.LogMessage.Level level) { switch (level) { case FATAL: - return LogLevel.FATAL; + return Level.SEVERE; case ERROR: - return LogLevel.ERROR; + return Level.SEVERE; case WARNING: return LogLevel.WARNING; case CONFIG: @@ -94,9 +94,9 @@ class ProtobufSerialization { case EVENT: return LogLevel.EVENT; case DEBUG: - return LogLevel.DEBUG; + return Level.FINE; case SPAM: - return LogLevel.SPAM; + return Level.FINEST; case UNKNOWN: case UNRECOGNIZED: default: @@ -106,9 +106,9 @@ class ProtobufSerialization { private static LogProtocol.LogMessage.Level toLogMessageLevel(Level level) { Level vespaLevel = LogLevel.getVespaLogLevel(level); - if (vespaLevel.equals(LogLevel.FATAL)) { + if (vespaLevel.equals(Level.SEVERE)) { return LogProtocol.LogMessage.Level.FATAL; - } else if (vespaLevel.equals(LogLevel.ERROR)) { + } else if (vespaLevel.equals(Level.SEVERE)) { return LogProtocol.LogMessage.Level.ERROR; } else if (vespaLevel.equals(LogLevel.WARNING)) { return LogProtocol.LogMessage.Level.WARNING; @@ -118,9 +118,9 @@ class ProtobufSerialization { return LogProtocol.LogMessage.Level.INFO; } else if (vespaLevel.equals(LogLevel.EVENT)) { return LogProtocol.LogMessage.Level.EVENT; - } else if (vespaLevel.equals(LogLevel.DEBUG)) { + } else if (vespaLevel.equals(Level.FINE)) { return LogProtocol.LogMessage.Level.DEBUG; - } else if (vespaLevel.equals(LogLevel.SPAM)) { + } else if (vespaLevel.equals(Level.FINEST)) { return LogProtocol.LogMessage.Level.SPAM; } else { return LogProtocol.LogMessage.Level.UNKNOWN; diff --git a/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java index 6549e06834d..fa9dbd0460e 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java @@ -68,7 +68,7 @@ public class SendProxyTestCase { sendMessage(0, null); assertNull(logHandler.trace); - log.setLevel(LogLevel.DEBUG); + log.setLevel(Level.FINE); sendMessage(0, null); assertNull(logHandler.trace); @@ -90,7 +90,7 @@ public class SendProxyTestCase { "\n", logHandler.trace); logHandler.trace = null; - log.setLevel(LogLevel.SPAM); + log.setLevel(Level.FINEST); sendMessage(1, null); assertNull(logHandler.trace); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index c0b4ecb1a36..0b5ddc6a943 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -99,11 +99,11 @@ public class AclMaintainer { dockerOperations.executeCommandInNetworkNamespace(context, ipVersion.iptablesRestore(), fileHandler.absolutePath()); } catch (Exception e) { if (flush) { - context.log(logger, LogLevel.ERROR, "Exception occurred while syncing iptable " + table + ", attempting rollback", e); + context.log(logger, Level.SEVERE, "Exception occurred while syncing iptable " + table + ", attempting rollback", e); try { dockerOperations.executeCommandInNetworkNamespace(context, ipVersion.iptablesCmd(), "-F", "-t", table); } catch (Exception ne) { - context.log(logger, LogLevel.ERROR, "Rollback of table " + table + " failed, giving up", ne); + context.log(logger, Level.SEVERE, "Rollback of table " + table + " failed, giving up", ne); } } else { context.log(logger, LogLevel.WARNING, "Unable to sync iptables for " + table, e); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 3cb74aef5a7..860cc8468e6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -109,7 +109,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { public boolean converge(NodeAgentContext context) { try { - context.log(logger, LogLevel.DEBUG, "Checking certificate"); + context.log(logger, Level.FINE, "Checking certificate"); Path containerSiaDirectory = context.pathOnHostFromPathInNode(CONTAINER_SIA_DIRECTORY); Path privateKeyFile = SiaUtils.getPrivateKeyFile(containerSiaDirectory, context.identity()); Path certificateFile = SiaUtils.getCertificateFile(containerSiaDirectory, context.identity()); @@ -146,7 +146,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { return true; } } - context.log(logger, LogLevel.DEBUG, "Certificate is still valid"); + context.log(logger, Level.FINE, "Certificate is still valid"); return false; } catch (IOException e) { throw new UncheckedIOException(e); @@ -169,7 +169,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { Instant expiry = certificate.getNotAfter().toInstant(); return Duration.between(now, expiry); } catch (IOException e) { - context.log(logger, LogLevel.ERROR, "Unable to read certificate at " + certificateFile, e); + context.log(logger, Level.SEVERE, "Unable to read certificate at " + certificateFile, e); return Duration.ZERO; } } @@ -236,14 +236,14 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { context.log(logger, "Instance successfully refreshed and credentials written to file"); } catch (ZtsClientException e) { if (e.getErrorCode() == 403 && e.getDescription().startsWith("Certificate revoked")) { - context.log(logger, LogLevel.ERROR, "Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e); + context.log(logger, Level.SEVERE, "Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e); registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile); } else { throw e; } } } catch (Exception e) { - context.log(logger, LogLevel.ERROR, "Certificate refresh failed: " + e.getMessage(), e); + context.log(logger, Level.SEVERE, "Certificate refresh failed: " + e.getMessage(), e); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index d473b3dcf40..bc532a84b64 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -164,7 +164,7 @@ public class NodeAgentImpl implements NodeAgent { void resumeNodeIfNeeded(NodeAgentContext context) { if (!hasResumedNode) { - context.log(logger, LogLevel.DEBUG, "Starting optional node program resume command"); + context.log(logger, Level.FINE, "Starting optional node program resume command"); dockerOperations.resumeNode(context); hasResumedNode = true; } @@ -404,10 +404,10 @@ public class NodeAgentImpl implements NodeAgent { context.log(logger, LogLevel.WARNING, "Container unexpectedly gone, resetting containerState to " + containerState); } catch (DockerException e) { numberOfUnhandledException++; - context.log(logger, LogLevel.ERROR, "Caught a DockerException", e); + context.log(logger, Level.SEVERE, "Caught a DockerException", e); } catch (Throwable e) { numberOfUnhandledException++; - context.log(logger, LogLevel.ERROR, "Unhandled exception, ignoring", e); + context.log(logger, Level.SEVERE, "Unhandled exception, ignoring", e); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index d2e214d9199..a3340e4cedb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -80,7 +80,7 @@ public class NodeRepositoryProvisioner implements Provisioner { @Override public List prepare(ApplicationId application, ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { - log.log(zone.system().isCd() ? Level.INFO : LogLevel.DEBUG, + log.log(zone.system().isCd() ? Level.INFO : Level.FINE, () -> "Received deploy prepare request for " + requested + " for application " + application + ", cluster " + cluster); diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java index d1e85adb503..666dfd47b47 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java @@ -121,7 +121,7 @@ public class TestRunnerHandler extends LoggingRequestHandler { return level.getName().equals("html") ? "html" : level.intValue() < LogLevel.INFO.intValue() ? "debug" : level.intValue() < LogLevel.WARNING.intValue() ? "info" - : level.intValue() < LogLevel.ERROR.intValue() ? "warning" + : level.intValue() < Level.SEVERE.intValue() ? "warning" : "error"; } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index eccafc92c34..2c06c4a296a 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -246,7 +246,7 @@ class ClientFeederV3 { if (result.isAccepted()) { outstandingOperations.incrementAndGet(); updateOpsPerSec(); - log(LogLevel.DEBUG, "Sent message successfully, document id: ", msg.get().getOperationId()); + log(Level.FINE, "Sent message successfully, document id: ", msg.get().getOperationId()); } else if (!result.getError().isFatal()) { repliesFromOldMessages.add(createOperationStatus(msg.get().getOperationId(), result.getError().getMessage(), @@ -297,7 +297,7 @@ class ClientFeederV3 { return null; } metric.add(MetricNames.NUM_OPERATIONS, 1, null /*metricContext*/); - log(LogLevel.DEBUG, "Successfully deserialized document id: ", message.getOperationId()); + log(Level.FINE, "Successfully deserialized document id: ", message.getOperationId()); return message; } -- cgit v1.2.3 From 89fa2fd9926579e6e2b24adf4281d063ecf62c43 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Fri, 24 Apr 2020 21:23:29 +0200 Subject: Replace remaining LogLevel. with corresponding Level --- .../yahoo/vespa/clustercontroller/core/EventLog.java | 4 ++-- .../clustercontroller/core/NodeStateGatherer.java | 18 +++++++++--------- .../clustercontroller/core/StateChangeHandler.java | 2 +- .../core/SystemStateBroadcaster.java | 2 +- .../core/status/statuspage/StatusPageServer.java | 6 +++--- .../config/server/deploy/DeployHandlerLogger.java | 2 +- .../vespa/config/server/rpc/GetConfigProcessor.java | 2 +- .../server/session/LocalSessionStateWatcher.java | 2 +- .../config/server/session/SessionPreparerTest.java | 2 +- .../container/jdisc/ThreadedHttpRequestHandler.java | 2 +- .../rendering/AsynchronousSectionedRenderer.java | 2 +- .../com/yahoo/container/jdisc/LoggingTestCase.java | 2 +- .../com/yahoo/search/cluster/ClusterSearcher.java | 10 +++++----- .../hosted/controller/api/integration/LogEntry.java | 6 +++--- .../controller/deployment/InternalStepRunner.java | 2 +- .../filedistribution/FileReferenceDownloader.java | 2 +- .../logserver/protocol/ProtobufSerialization.java | 20 ++++++++++---------- .../java/com/yahoo/messagebus/SendProxyTestCase.java | 2 +- .../hosted/node/admin/component/TaskContext.java | 4 ++-- .../node/admin/maintenance/StorageMaintainer.java | 2 +- .../node/admin/maintenance/acl/AclMaintainer.java | 2 +- .../identity/AthenzCredentialsMaintainer.java | 2 +- .../hosted/node/admin/nodeagent/NodeAgentImpl.java | 10 +++++----- .../orchestrator/status/ZkStatusServiceTest.java | 2 +- .../vespa/hosted/testrunner/TestRunnerHandler.java | 4 ++-- 25 files changed, 57 insertions(+), 57 deletions(-) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java index 4af4b2b15a1..367771ff87a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java @@ -57,9 +57,9 @@ public class EventLog implements EventLogInterface { } if (e instanceof NodeEvent) { - addNodeOnlyEvent((NodeEvent)e, logInfo ? LogLevel.INFO: Level.FINE); + addNodeOnlyEvent((NodeEvent)e, logInfo ? Level.INFO: Level.FINE); } else { - log.log(logInfo ? LogLevel.INFO : Level.FINE, e.toString()); + log.log(logInfo ? Level.INFO : Level.FINE, e.toString()); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java index 87a572ce4a0..da32caa0c4e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java @@ -157,7 +157,7 @@ public class NodeStateGatherer { if (req.getReply().getReturnCode() == ErrorCode.TIMEOUT) { String msg = "RPC timeout"; if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + "RPC timeout talking to node.", NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + "RPC timeout talking to node.", NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } @@ -172,7 +172,7 @@ public class NodeStateGatherer { if (msg.equals("Connection refused")) { msg = "Connection error: Connection refused"; if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); @@ -184,7 +184,7 @@ public class NodeStateGatherer { msg += " Node is no longer in slobrok."; } if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } @@ -192,14 +192,14 @@ public class NodeStateGatherer { } else if (msg.equals("Connection timed out")) { if (info.getReportedState().getState().oneOf("ui")) { msg = "Connection error: Timeout"; - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else { log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } } else { msg = "Connection error: " + reason; if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } @@ -210,7 +210,7 @@ public class NodeStateGatherer { req.getReply().getReturnCode() + ": " + req.getReply().getReturnMessage(); if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } @@ -221,7 +221,7 @@ public class NodeStateGatherer { } else if (req.getReply().getReturnCode() == ErrorCode.NO_SUCH_METHOD) { String msg = "no such RPC method error"; if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } @@ -233,7 +233,7 @@ public class NodeStateGatherer { log.log(Level.FINE, "Failed to get node state from " + info + " because it is still shutting down."); } else { if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } @@ -242,7 +242,7 @@ public class NodeStateGatherer { } else { String msg = "Got unexpected error, assumed to be node issue " + req.getReply().getReturnCode() + ": " + req.getReply().getReturnMessage(); if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index abdb6255ca2..18e548f6f1c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -132,7 +132,7 @@ public class StateChangeHandler { // *** LOGGING ONLY if ( ! reportedState.similarTo(node.getReportedState())) { if (reportedState.getState().equals(State.DOWN)) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Failed to get node state: " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Failed to get node state: " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else { eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Now reporting state " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), Level.FINE); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java index 95e03d1f6f1..502a0823b13 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java @@ -66,7 +66,7 @@ public class SystemStateBroadcaster { long time = timer.getCurrentTimeInMillis(); Long lastReported = lastErrorReported.get(info.getNode()); boolean alreadySeen = (lastReported != null && time - lastReported < minTimeBetweenNodeErrorLogging); - log.log((nodeOk && !alreadySeen) ? LogLevel.WARNING : Level.FINE, message); + log.log((nodeOk && !alreadySeen) ? Level.WARNING : Level.FINE, message); if (!alreadySeen) { lastErrorReported.put(info.getNode(), time); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index 3028ffe6e2c..3d3de32c356 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -146,7 +146,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { } catch (SocketTimeoutException e) { // Ignore, since timeout is set to 100 ms } catch (java.io.IOException e) { - log.log(shouldBeConnected ? LogLevel.WARNING : Level.FINE, "Caught IO exception in ServerSocket.accept(): " + e.getMessage()); + log.log(shouldBeConnected ? Level.WARNING : Level.FINE, "Caught IO exception in ServerSocket.accept(): " + e.getMessage()); } if (connection == null) continue; log.log(Level.FINE, "Got a status page request."); @@ -235,7 +235,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { output.write(header.toString().getBytes()); output.write(response.getOutputStream().toByteArray()); } catch (java.io.IOException e) { - log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? Level.FINE : LogLevel.INFO, + log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? Level.FINE : Level.INFO, "Failed to process HTTP request : " + e.getMessage()); } catch (Exception e) { log.log(Level.WARNING, "Caught exception in HTTP server thread: " @@ -244,7 +244,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { if (output != null) try { output.close(); } catch (IOException e) { - log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? Level.FINE : LogLevel.INFO, + log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? Level.FINE : Level.INFO, "Failed to close output stream on socket " + connection + ": " + e.getMessage()); } if (connection != null) try{ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index f45e2cf6285..ced73cda6fb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -33,7 +33,7 @@ public class DeployHandlerLogger implements DeployLogger { @Override public void log(Level level, String message) { - if ((level == LogLevel.FINE || + if ((level == Level.FINE || level == Level.FINE || level == Level.FINEST) && !verbose) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index cf6890e22d2..7809000695a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -49,7 +49,7 @@ class GetConfigProcessor implements Runnable { private void respond(JRTServerConfigRequest request) { Request req = request.getRequest(); if (req.isError()) { - Level logLevel = (req.errorCode() == ErrorCode.APPLICATION_NOT_LOADED) ? Level.FINE : LogLevel.INFO; + Level logLevel = (req.errorCode() == ErrorCode.APPLICATION_NOT_LOADED) ? Level.FINE : Level.INFO; log.log(logLevel, logPre + req.errorMessage()); } rpcServer.respond(request); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java index eb5770972ff..662094fc0ca 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java @@ -37,7 +37,7 @@ public class LocalSessionStateWatcher { // Will delete session if it exists in local session repo private void sessionChanged(Session.Status status) { long sessionId = session.getSessionId(); - log.log(status == Session.Status.DELETE ? LogLevel.INFO : Level.FINE, + log.log(status == Session.Status.DELETE ? Level.INFO : Level.FINE, session.logPre() + "Session change: Local session " + sessionId + " changed status to " + status); if (status.equals(Session.Status.DELETE) && localSessionRepo.getSession(sessionId) != null) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 287996f6bec..cdf57b68814 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -176,7 +176,7 @@ public class SessionPreparerTest { final StringBuilder logged = new StringBuilder(); DeployLogger logger = (level, message) -> { System.out.println(level + ": "+message); - if (level.equals(LogLevel.WARNING) && message.contains("The host mytesthost is already in use")) logged.append("ok"); + if (level.equals(Level.WARNING) && message.contains("The host mytesthost is already in use")) logged.append("ok"); }; preparer.prepare(ctx, logger, new PrepareParams.Builder().build(), Optional.empty(), tenantPath, Instant.now()); assertEquals(logged.toString(), ""); diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 9eec85f58b6..2a4fc66dc27 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -118,7 +118,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler catch (IOException e) { metric.add(RENDERING_ERRORS, 1, null); long time = System.currentTimeMillis() - startTime; - log.log(time < 900 ? LogLevel.INFO : LogLevel.WARNING, + log.log(time < 900 ? Level.INFO : LogLevel.WARNING, "IO error while responding to " + " [" + request.getUri() + "] " + "(total time " + time + " ms) ", e); diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java index 6c5ac7718d5..ba246f26639 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java @@ -527,7 +527,7 @@ public abstract class AsynchronousSectionedRenderer e try { render(); } catch (Exception e) { - Level level = LogLevel.WARNING; + Level level = Level.WARNING; if ((e instanceof IOException)) { level = Level.FINE; if ( ! clientClosed) { diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java index 876fd0d2b57..22e55eb5291 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java @@ -101,7 +101,7 @@ public class LoggingTestCase { stream.send(createData()); stream.send(createData()); stream.flush(); - assertNull(logChecker.errorCounter.get(LogLevel.INFO)); + assertNull(logChecker.errorCounter.get(Level.INFO)); assertEquals(1, logChecker.errorCounter.get(Level.FINE).intValue()); assertEquals(2, logChecker.errorCounter.get(Level.FINEST).intValue()); } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 3afeb35ad2d..db79c610dc7 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -71,7 +71,7 @@ public abstract class ClusterSearcher extends PingableSearcher implements Nod /** Pinging a node, called from ClusterMonitor */ @Override public final void ping(ClusterMonitor clusterMonitor, T p, Executor executor) { - log(LogLevel.FINE, "Sending ping to: ", p); + log(Level.FINE, "Sending ping to: ", p); Pinger pinger = new Pinger(p); FutureTask future = new FutureTask<>(pinger); @@ -97,10 +97,10 @@ public abstract class ClusterSearcher extends PingableSearcher implements Nod if (pong.badResponse()) { clusterMonitor.failed(p, pong.error().get()); - log(LogLevel.FINE, "Failed ping - ", pong); + log(Level.FINE, "Failed ping - ", pong); } else { clusterMonitor.responded(p); - log(LogLevel.FINE, "Answered ping - ", p); + log(Level.FINE, "Answered ping - ", p); } if (logThrowable != null) { @@ -173,7 +173,7 @@ public abstract class ClusterSearcher extends PingableSearcher implements Nod if (result.hits().getError().getCode() == Error.TIMEOUT.code) return result; // Retry is unlikely to help - log(LogLevel.FINER, "No result, checking for timeout."); + log(Level.FINER, "No result, checking for timeout."); tries++; connection = nodes.select(code, tries); } while (tries < nodes.getNodeCount()); @@ -211,7 +211,7 @@ public abstract class ClusterSearcher extends PingableSearcher implements Nod try { result = search(query, execution, connection); } catch (RuntimeException e) { //TODO: Exceptions should not be used to signal backend communication errors - log(LogLevel.WARNING, "An exception occurred while invoking backend searcher.", e); + log(Level.WARNING, "An exception occurred while invoking backend searcher.", e); result = new Result(query, ErrorMessage.createBackendCommunicationError("Failed calling " + connection + " in " + this + " for " + query + ": " + Exceptions.toMessageString(e))); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java index c16e72e3ad1..49813474315 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java @@ -63,7 +63,7 @@ public class LogEntry { .filter(parts -> parts.length == 7) .map(parts -> new LogEntry(0, Instant.EPOCH.plus((long) (Double.parseDouble(parts[0]) * 1_000_000), ChronoUnit.MICROS), - typeOf(LogLevel.parse(parts[5])), + typeOf(Level.parse(parts[5])), parts[1] + '\t' + parts[3] + '\t' + parts[4] + '\n' + parts[6].replaceAll("\\\\n", "\n") .replaceAll("\\\\t", "\t"))) @@ -103,8 +103,8 @@ public class LogEntry { } public static Type typeOf(Level level) { - return level.intValue() < LogLevel.INFO.intValue() ? Type.debug - : level.intValue() < LogLevel.WARNING.intValue() ? Type.info + return level.intValue() < Level.INFO.intValue() ? Type.debug + : level.intValue() < Level.WARNING.intValue() ? Type.info : level.intValue() < Level.SEVERE.intValue() ? Type.warning : Type.error; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 58547edc52a..ab4f76b5a92 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -216,7 +216,7 @@ public class InternalStepRunner implements StepRunner { logger.logAll(prepareResponse.log.stream() .map(entry -> new LogEntry(0, // Sequenced by BufferedLogStore. Instant.ofEpochMilli(entry.time), - LogEntry.typeOf(LogLevel.parse(entry.level)), + LogEntry.typeOf(Level.parse(entry.level)), entry.message)) .collect(toList())); if ( ! prepareResponse.configChangeActions.refeedActions.stream().allMatch(action -> action.allowed)) { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 78afcd72056..4ee9eacc2be 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -117,7 +117,7 @@ public class FileReferenceDownloader { request.parameters().add(new Int32Value(fileReferenceDownload.downloadFromOtherSourceIfNotFound() ? 0 : 1)); connection.invokeSync(request, (double) rpcTimeout.getSeconds()); - Level logLevel = (retryCount > 0 ? LogLevel.INFO : Level.FINE); + Level logLevel = (retryCount > 0 ? Level.INFO : Level.FINE); if (validateResponse(request)) { log.log(logLevel, () -> "Request callback, OK. Req: " + request + "\nSpec: " + connection + ", retry count " + retryCount); if (request.returnValues().get(0).asInt32() == 0) { diff --git a/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java index 3f040821a7f..9065b47fa52 100644 --- a/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java +++ b/logserver/src/main/java/ai/vespa/logserver/protocol/ProtobufSerialization.java @@ -3,12 +3,12 @@ package ai.vespa.logserver.protocol; import ai.vespa.logserver.protocol.protobuf.LogProtocol; import com.google.protobuf.InvalidProtocolBufferException; -import java.util.logging.Level; +import com.yahoo.log.LogLevel; import com.yahoo.log.LogMessage; import java.time.Instant; import java.util.List; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import static java.util.stream.Collectors.toList; @@ -82,9 +82,9 @@ class ProtobufSerialization { private static Level fromLogMessageLevel(LogProtocol.LogMessage.Level level) { switch (level) { case FATAL: - return Level.SEVERE; + return LogLevel.FATAL; case ERROR: - return Level.SEVERE; + return LogLevel.ERROR; case WARNING: return LogLevel.WARNING; case CONFIG: @@ -94,9 +94,9 @@ class ProtobufSerialization { case EVENT: return LogLevel.EVENT; case DEBUG: - return Level.FINE; + return LogLevel.DEBUG; case SPAM: - return Level.FINEST; + return LogLevel.SPAM; case UNKNOWN: case UNRECOGNIZED: default: @@ -106,9 +106,9 @@ class ProtobufSerialization { private static LogProtocol.LogMessage.Level toLogMessageLevel(Level level) { Level vespaLevel = LogLevel.getVespaLogLevel(level); - if (vespaLevel.equals(Level.SEVERE)) { + if (vespaLevel.equals(LogLevel.FATAL)) { return LogProtocol.LogMessage.Level.FATAL; - } else if (vespaLevel.equals(Level.SEVERE)) { + } else if (vespaLevel.equals(LogLevel.ERROR)) { return LogProtocol.LogMessage.Level.ERROR; } else if (vespaLevel.equals(LogLevel.WARNING)) { return LogProtocol.LogMessage.Level.WARNING; @@ -118,9 +118,9 @@ class ProtobufSerialization { return LogProtocol.LogMessage.Level.INFO; } else if (vespaLevel.equals(LogLevel.EVENT)) { return LogProtocol.LogMessage.Level.EVENT; - } else if (vespaLevel.equals(Level.FINE)) { + } else if (vespaLevel.equals(LogLevel.DEBUG)) { return LogProtocol.LogMessage.Level.DEBUG; - } else if (vespaLevel.equals(Level.FINEST)) { + } else if (vespaLevel.equals(LogLevel.SPAM)) { return LogProtocol.LogMessage.Level.SPAM; } else { return LogProtocol.LogMessage.Level.UNKNOWN; diff --git a/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java index fa9dbd0460e..48b0d355e87 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java @@ -64,7 +64,7 @@ public class SendProxyTestCase { LogHandler logHandler = new LogHandler(); log.addHandler(logHandler); - log.setLevel(LogLevel.INFO); + log.setLevel(Level.INFO); sendMessage(0, null); assertNull(logHandler.trace); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java index 74b47a49d2e..b84e25f59ed 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java @@ -28,7 +28,7 @@ public interface TaskContext { } /** - * Log message at LogLevel.INFO, scoped to denote the current task. The message may + * Log message at Level.INFO, scoped to denote the current task. The message may * also be directed to status pages or similar. * * Please do not call this too many times as that spams the log. Typically a task may call @@ -37,7 +37,7 @@ public interface TaskContext { * Do not log a message that is also recorded with recordSystemModification. */ default void log(Logger logger, String message) { - log(logger, LogLevel.INFO, message); + log(logger, Level.INFO, message); } default void log(Logger logger, String messageFormat, Object... args) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 16d0d823ac2..ef515022ae6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -82,7 +82,7 @@ public class StorageMaintainer { diskUsage.put(context.containerName(), diskUsageBytes); return Optional.of(diskUsageBytes); } catch (Exception e) { - context.log(logger, LogLevel.WARNING, "Failed to get disk usage", e); + context.log(logger, Level.WARNING, "Failed to get disk usage", e); return Optional.empty(); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index 0b5ddc6a943..360cea8a60d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -106,7 +106,7 @@ public class AclMaintainer { context.log(logger, Level.SEVERE, "Rollback of table " + table + " failed, giving up", ne); } } else { - context.log(logger, LogLevel.WARNING, "Unable to sync iptables for " + table, e); + context.log(logger, Level.WARNING, "Unable to sync iptables for " + table, e); } } }; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 860cc8468e6..3320851a36c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -136,7 +136,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { if (shouldRefreshCredentials(age)) { context.log(logger, "Certificate is ready to be refreshed (age=%s)", age.toString()); if (shouldThrottleRefreshAttempts(context.containerName(), now)) { - context.log(logger, LogLevel.WARNING, String.format( + context.log(logger, Level.WARNING, String.format( "Skipping refresh attempt as last refresh was on %s (less than %s ago)", lastRefreshAttempt.get(context.containerName()).toString(), REFRESH_BACKOFF.toString())); return false; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index bc532a84b64..eb3bd0d6d2e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -290,7 +290,7 @@ public class NodeAgentImpl implements NodeAgent { } catch (RuntimeException e) { // It's bad to continue as-if nothing happened, but on the other hand if we do not proceed to // remove container, we will not be able to upgrade to fix any problems in the suspend logic! - context.log(logger, LogLevel.WARNING, "Failed trying to suspend container", e); + context.log(logger, Level.WARNING, "Failed trying to suspend container", e); } } @@ -340,7 +340,7 @@ public class NodeAgentImpl implements NodeAgent { } stopServices(context); } catch (Exception e) { - context.log(logger, LogLevel.WARNING, "Failed stopping services, ignoring", e); + context.log(logger, Level.WARNING, "Failed stopping services, ignoring", e); } } @@ -396,12 +396,12 @@ public class NodeAgentImpl implements NodeAgent { public void converge(NodeAgentContext context) { try { doConverge(context); - context.log(logger, LogLevel.INFO, "Converged"); + context.log(logger, Level.INFO, "Converged"); } catch (ConvergenceException e) { context.log(logger, e.getMessage()); } catch (ContainerNotFoundException e) { containerState = ABSENT; - context.log(logger, LogLevel.WARNING, "Container unexpectedly gone, resetting containerState to " + containerState); + context.log(logger, Level.WARNING, "Container unexpectedly gone, resetting containerState to " + containerState); } catch (DockerException e) { numberOfUnhandledException++; context.log(logger, Level.SEVERE, "Caught a DockerException", e); @@ -510,7 +510,7 @@ public class NodeAgentImpl implements NodeAgent { StringBuilder builder = new StringBuilder(); appendIfDifferent(builder, "state", lastNode, node, NodeSpec::state); if (builder.length() > 0) { - context.log(logger, LogLevel.INFO, "Changes to node: " + builder.toString()); + context.log(logger, Level.INFO, "Changes to node: " + builder.toString()); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java index fc025d013c4..02c336e4109 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java @@ -82,7 +82,7 @@ public class ZkStatusServiceTest { @Before public void setUp() throws Exception { - Logger.getLogger("").setLevel(LogLevel.WARNING); + Logger.getLogger("").setLevel(Level.WARNING); testingServer = new TestingServer(); curator = createConnectedCurator(testingServer); diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java index 666dfd47b47..e92dbcede5a 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java @@ -119,8 +119,8 @@ public class TestRunnerHandler extends LoggingRequestHandler { public static String typeOf(Level level) { return level.getName().equals("html") ? "html" - : level.intValue() < LogLevel.INFO.intValue() ? "debug" - : level.intValue() < LogLevel.WARNING.intValue() ? "info" + : level.intValue() < Level.INFO.intValue() ? "debug" + : level.intValue() < Level.WARNING.intValue() ? "info" : level.intValue() < Level.SEVERE.intValue() ? "warning" : "error"; } -- cgit v1.2.3 From c0c9169385861c23ba640212ebb1b3beecd555d5 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Sat, 25 Apr 2020 00:09:00 +0200 Subject: Use correct log Level class where search & replace has failed. --- .../com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java | 2 +- .../src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java | 2 +- .../java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java | 2 +- .../java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java | 2 +- metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java | 4 ++-- .../src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index 18e548f6f1c..cb6bacf97a6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -118,7 +118,7 @@ public class StateChangeHandler { final NodeStateOrHostInfoChangeHandler nodeListener) { final NodeState currentState = currentClusterState.getNodeState(node.getNode()); - final LogLevel level = (currentState.equals(reportedState) && node.getVersion() == 0) ? Level.FINEST : Level.FINE; + final Level level = (currentState.equals(reportedState) && node.getVersion() == 0) ? Level.FINEST : Level.FINE; if (log.isLoggable(level)) { log.log(level, String.format("Got nodestate reply from %s: %s (Current state is %s)", node, node.getReportedState().getTextualDifference(reportedState), currentState.toString(true))); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 8082b340970..3cad8def541 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -296,7 +296,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { log.log(Level.FINE, logPre + "Finished reloading " + responsesSent + " requests"); } - private void logRequestDebug(LogLevel level, String message, JRTServerConfigRequest request) { + private void logRequestDebug(Level level, String message, JRTServerConfigRequest request) { if (log.isLoggable(level)) { log.log(level, message + request.getShortDescription()); } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 2a4fc66dc27..3a99ee7d0c6 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -118,7 +118,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler catch (IOException e) { metric.add(RENDERING_ERRORS, 1, null); long time = System.currentTimeMillis() - startTime; - log.log(time < 900 ? Level.INFO : LogLevel.WARNING, + log.log(time < 900 ? Level.INFO : Level.WARNING, "IO error while responding to " + " [" + request.getUri() + "] " + "(total time " + time + " ms) ", e); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 56b5a826cba..8bd7bdac8a5 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -300,7 +300,7 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { String expUserId = query.properties().getString(streamingUserid); String expGroupName = query.properties().getString(streamingGroupname); - LogLevel logLevel = Level.SEVERE; + Level logLevel = Level.SEVERE; if (skippedEarlierResult) { logLevel = Level.FINE; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java index e634d533e5c..30e7d7b81f8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java @@ -197,9 +197,9 @@ public class RpcServer { } private void logSpentTime() { - Level logLevel = DEBUG; + Level logLevel = Level.FINE; if (spentTime() > LOG_SPENT_TIME_LIMIT) { - logLevel = INFO; + logLevel = Level.INFO; } if (log.isLoggable(logLevel)) { log.log(logLevel, "RPC request '" + request.methodName() + "' with parameters '" + diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index 2c06c4a296a..db1c2471752 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -325,7 +325,7 @@ class ClientFeederV3 { } } - protected final void log(LogLevel level, Object... msgParts) { + protected final void log(Level level, Object... msgParts) { StringBuilder s; if (!log.isLoggable(level)) { -- cgit v1.2.3 From 0ded26c5c4fddcb22bf7566f2c731533e4a7ddcb Mon Sep 17 00:00:00 2001 From: gjoranv Date: Sat, 25 Apr 2020 01:42:52 +0200 Subject: Use Vespa LogLevel where appropriate. --- .../com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java | 7 ++++--- .../yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java | 6 ++++-- .../yahoo/vespa/hosted/controller/api/integration/LogEntry.java | 5 +++-- .../vespa/hosted/controller/deployment/InternalStepRunner.java | 4 ++-- .../ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java | 4 +++- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index ced73cda6fb..c33c5ff9f57 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -3,7 +3,8 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; -import java.util.logging.Level; + +import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -34,8 +35,8 @@ public class DeployHandlerLogger implements DeployLogger { @Override public void log(Level level, String message) { if ((level == Level.FINE || - level == Level.FINE || - level == Level.FINEST) && + level == LogLevel.DEBUG || + level == LogLevel.SPAM) && !verbose) { return; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java index 078836401ac..6aa72e3e672 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import java.util.logging.Level; + +import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; @@ -42,8 +44,8 @@ public class DeployHandlerLoggerTest { } private void logMessages(DeployLogger logger) { - logger.log(Level.FINE, "foobar"); - logger.log(Level.FINEST, "foobar"); + logger.log(LogLevel.DEBUG, "foobar"); + logger.log(LogLevel.SPAM, "foobar"); logger.log(Level.FINE, "baz"); logger.log(Level.WARNING, "baz"); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java index 49813474315..02c6972b543 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java @@ -1,6 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration; +import com.yahoo.log.LogLevel; + import java.util.logging.Level; import java.io.BufferedReader; @@ -12,7 +14,6 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Objects; -import java.util.logging.Level; import java.util.stream.Collectors; import static java.nio.charset.StandardCharsets.UTF_8; @@ -63,7 +64,7 @@ public class LogEntry { .filter(parts -> parts.length == 7) .map(parts -> new LogEntry(0, Instant.EPOCH.plus((long) (Double.parseDouble(parts[0]) * 1_000_000), ChronoUnit.MICROS), - typeOf(Level.parse(parts[5])), + typeOf(LogLevel.parse(parts[5])), parts[1] + '\t' + parts[3] + '\t' + parts[4] + '\n' + parts[6].replaceAll("\\\\n", "\n") .replaceAll("\\\\t", "\t"))) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index ab4f76b5a92..bc206011bb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -16,7 +16,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; -import java.util.logging.Level; +import com.yahoo.log.LogLevel; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; @@ -216,7 +216,7 @@ public class InternalStepRunner implements StepRunner { logger.logAll(prepareResponse.log.stream() .map(entry -> new LogEntry(0, // Sequenced by BufferedLogStore. Instant.ofEpochMilli(entry.time), - LogEntry.typeOf(Level.parse(entry.level)), + LogEntry.typeOf(LogLevel.parse(entry.level)), entry.message)) .collect(toList())); if ( ! prepareResponse.configChangeActions.refeedActions.stream().allMatch(action -> action.allowed)) { diff --git a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java index 1dd79316eda..2b702058ede 100644 --- a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java +++ b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java @@ -11,6 +11,8 @@ import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.Values; import java.util.logging.Level; + +import com.yahoo.log.LogLevel; import com.yahoo.log.LogMessage; import com.yahoo.logserver.LogDispatcher; import org.junit.Test; @@ -31,7 +33,7 @@ import static org.mockito.Mockito.verify; public class ArchiveLogMessagesMethodTest { private static final LogMessage MESSAGE_1 = - LogMessage.of(Instant.EPOCH.plus(1000, ChronoUnit.DAYS), "localhost", 12, 3456, "my-service", "my-component", Level.SEVERE, "My error message"); + LogMessage.of(Instant.EPOCH.plus(1000, ChronoUnit.DAYS), "localhost", 12, 3456, "my-service", "my-component", LogLevel.ERROR, "My error message"); private static final LogMessage MESSAGE_2 = LogMessage.of(Instant.EPOCH.plus(5005, ChronoUnit.DAYS), "localhost", 12, 6543, "my-service", "my-component", Level.INFO, "My info message"); -- cgit v1.2.3 From ff556e3da8ca8c79112bf36315c42c8800c6b25d Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Sat, 25 Apr 2020 13:48:05 +0200 Subject: Revert "Optimize B-tree bucket DB lookup with used-bits aggregation" --- .../src/tests/distributor/bucketdatabasetest.cpp | 33 ----------------- .../storage/bucketdb/btree_bucket_database.cpp | 41 ++++++---------------- .../vespa/storage/bucketdb/btree_bucket_database.h | 17 +-------- 3 files changed, 12 insertions(+), 79 deletions(-) diff --git a/storage/src/tests/distributor/bucketdatabasetest.cpp b/storage/src/tests/distributor/bucketdatabasetest.cpp index 56e996abf99..1d5355df92f 100644 --- a/storage/src/tests/distributor/bucketdatabasetest.cpp +++ b/storage/src/tests/distributor/bucketdatabasetest.cpp @@ -707,37 +707,4 @@ TEST_P(BucketDatabaseTest, DISABLED_benchmark_const_iteration) { db().toString(false).c_str(), elapsed); } -TEST_P(BucketDatabaseTest, DISABLED_benchmark_find_parents) { - constexpr uint32_t superbuckets = 1u << 16u; - constexpr uint32_t sub_buckets = 14; - constexpr uint32_t n_buckets = superbuckets * sub_buckets; - - std::vector bucket_keys; - bucket_keys.reserve(n_buckets); - - for (uint32_t sb = 0; sb < superbuckets; ++sb) { - for (uint64_t i = 0; i < sub_buckets; ++i) { - document::BucketId bucket(48, (i << 32ULL) | sb); // TODO eval with different bit counts - bucket_keys.emplace_back(bucket.toKey()); - } - } - fprintf(stderr, "Inserting %zu buckets into DB\n", bucket_keys.size()); - std::sort(bucket_keys.begin(), bucket_keys.end()); - for (uint64_t k : bucket_keys) { - db().update(BucketDatabase::Entry(BucketId(BucketId::keyToBucketId(k)), BI3(0, 1, 2))); - } - - fprintf(stderr, "Invoking getParents() %zu times\n", bucket_keys.size()); - auto elapsed = vespalib::BenchmarkTimer::benchmark([&] { - std::vector entries; - for (uint64_t k : bucket_keys) { - db().getParents(BucketId(BucketId::keyToBucketId(k)), entries); - assert(entries.size() == 1); - entries.clear(); - } - }, 30); - fprintf(stderr, "Looking up all buckets in %s takes %g seconds\n", - db().toString(false).c_str(), elapsed); -} - } diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp index 2adb3049853..85f5c8c5be9 100644 --- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp +++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp @@ -194,27 +194,13 @@ void BTreeBucketDatabase::remove(const BucketId& bucket) { * This allows us to reuse a single iterator and to continue seeking forwards * from its current position. * - * To speed up the process of converging on the target bucket without needing - * to check many unrelated subtrees, we let the underlying B-tree automatically - * aggregate the min/max range of the used-bits of all contained bucket keys. - * If we e.g. know that the minimum number of used bits in the DB is 16, we can - * immediately seek to this level in the tree instead of working our way down - * one bit at a time. By definition, no parents can exist above this level. - * This is a very important optimization, as bucket trees are usually very well - * balanced due to randomized distribution of data (combined with a cluster-wide - * minimum tree level imposed by distribution bits). It is common that the minimum - * number of used bits == max number of used bits, i.e. a totally even split. - * This means that for a system without inconsistently split buckets (i.e. no - * parents) we're highly likely to converge on the target bucket in a single seek. - * * Algorithm: * * Core invariant: every subsequent iterator seek performed in this algorithm * is for a key that is strictly higher than the one the iterator is currently at. * * 1. Lbound seek to the lowest key that is known to exclude all already visited - * parents. On the first iteration we use a bit count equal to the minimum number - * of key used-bits in the entire DB, allowing us to potentially skip most subtrees. + * parents. On the first iteration this is zero, i.e. the first in-order node. * 2. If the current node's key is greater than that of the requested bucket's key, * we've either descended to--or beyond--it in its own subtree or we've entered * a disjoint subtree. Since we know that all parents must sort before any given @@ -243,13 +229,14 @@ void BTreeBucketDatabase::remove(const BucketId& bucket) { * 6. Iff iterator is still valid, go to step 2 * * This algorithm is able to skip through large parts of the tree in a sparsely populated - * tree, but the number of seeks will trend towards O(b - min_bits) as with the legacy - * implementation when a tree is densely populated (where `b` is the used-bits count of the - * most specific node in the tree for the target bucket, and min_bits is the minimum number - * of used-bits for any key in the database). This because all logical inner nodes in the tree - * will have subtrees under them. Even in the worst case we should be more efficient than the - * legacy Judy-based implementation since we've cut any dense search space in half for each - * invocation of seek() on the iterator. + * tree, but the number of seeks will trend towards O(b) as with the legacy implementation + * when a tree is densely populated. This because all logical inner nodes in the tree will + * have subtrees under them. Even in the worst case we should be more efficient than the + * legacy implementation since we've cut any dense search space in half for each invocation + * of seek() on the iterator + * + * TODO use min-aggregation to infer used bit range in the tree. This should allow us to + * skip scanning through many disjoint subtrees. */ BTreeBucketDatabase::BTree::ConstIterator BTreeBucketDatabase::find_parents_internal(const BTree::FrozenView& frozen_view, @@ -258,15 +245,9 @@ BTreeBucketDatabase::find_parents_internal(const BTree::FrozenView& frozen_view, { const uint64_t bucket_key = bucket.toKey(); auto iter = frozen_view.begin(); - if (!iter.valid()) { - return iter; - } - assert(iter.getAggregated().getMin() >= static_cast(BucketId::minNumBits)); - assert(iter.getAggregated().getMax() <= static_cast(BucketId::maxNumBits)); - // Start at the lowest possible tree level no parents can exist above, - // descending towards the bucket itself. + // Start at the root level, descending towards the bucket itself. // Try skipping as many levels of the tree as possible as we go. - uint32_t bits = iter.getAggregated().getMin(); + uint32_t bits = 1; while (iter.valid() && (iter.getKey() < bucket_key)) { auto candidate = BucketId(BucketId::keyToBucketId(iter.getKey())); if (candidate.contains(bucket)) { diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.h b/storage/src/vespa/storage/bucketdb/btree_bucket_database.h index 75736e59092..8898b0c395a 100644 --- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.h +++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.h @@ -4,8 +4,6 @@ #include "bucketdatabase.h" #include -#include -#include #include namespace storage { @@ -25,21 +23,8 @@ namespace storage { */ // TODO create and use a new DB interface with better bulk loading, snapshot and iteration support class BTreeBucketDatabase : public BucketDatabase { - - struct KeyUsedBitsMinMaxAggrCalc : search::btree::MinMaxAggrCalc { - constexpr static bool aggregate_over_values() { return false; } - constexpr static int32_t getVal(uint64_t key) noexcept { - static_assert(document::BucketId::CountBits == 6u); - return static_cast(key & 0b11'1111U); // 6 LSB of key contains used-bits - } - }; - // Mapping from u64: bucket key -> - using BTree = search::btree::BTree, - search::btree::BTreeDefaultTraits, - KeyUsedBitsMinMaxAggrCalc>; + using BTree = search::btree::BTree; using ReplicaStore = search::datastore::ArrayStore; using GenerationHandler = vespalib::GenerationHandler; -- cgit v1.2.3 From 9093fa66b3588e32573c57e2322da4b18e1f8c8d Mon Sep 17 00:00:00 2001 From: gjoranv Date: Sat, 25 Apr 2020 14:12:59 +0200 Subject: Update StemmingSearcher after rebasing to master. --- .../java/com/yahoo/prelude/querytransform/StemmingSearcher.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index af473dd7628..318912eab04 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -184,12 +184,12 @@ public class StemmingSearcher extends Searcher { Substring substring = getOffsets(current); if (segments.size() == 1) { - getLogger().log(LogLevel.DEBUG, () -> "Stem '"+current.stringValue()+"' mode "+index.getStemMode() + getLogger().log(Level.FINE, () -> "Stem '"+current.stringValue()+"' mode "+index.getStemMode() +" and language '"+context.language+"' -> '"+segments.get(0)+"'"); TaggableItem w = singleWordSegment(current, segments.get(0), index, substring, context.insidePhrase); setMetaData(current, context.reverseConnectivity, w); return (Item) w; - } else if (getLogger().isLoggable(LogLevel.DEBUG)) { + } else if (getLogger().isLoggable(Level.FINE)) { var buf = new StringBuilder(); buf.append("Stem '").append(current.stringValue()); buf.append("' mode ").append(index.getStemMode()); @@ -197,7 +197,7 @@ public class StemmingSearcher extends Searcher { for (StemList segment : segments) { buf.append(" '").append(segment).append("'"); } - getLogger().log(LogLevel.DEBUG, buf.toString()); + getLogger().log(Level.FINE, buf.toString()); } if (context.isCJK) @@ -206,7 +206,7 @@ public class StemmingSearcher extends Searcher { composite = chooseComposite(current, ((Item) current).getParent(), indexName); for (StemList segment : segments) { - getLogger().log(LogLevel.DEBUG, () -> "Stem to multiple segments '"+segment+"'"); + getLogger().log(Level.FINE, () -> "Stem to multiple segments '"+segment+"'"); TaggableItem w = singleWordSegment(current, segment, index, substring, context.insidePhrase); if (composite instanceof AndSegmentItem) { -- cgit v1.2.3 From 5e9b98aecac6a8919a22b6e73861e6f2af07df72 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Sat, 25 Apr 2020 19:50:07 +0200 Subject: Extend BTree frozen view with empty() and getAggregated() methods. --- vespalib/src/tests/btree/btreeaggregation_test.cpp | 20 ++++++++++++++++++++ vespalib/src/tests/btree/frozenbtree_test.cpp | 2 ++ vespalib/src/vespa/vespalib/btree/btreeroot.h | 6 ++++++ 3 files changed, 28 insertions(+) diff --git a/vespalib/src/tests/btree/btreeaggregation_test.cpp b/vespalib/src/tests/btree/btreeaggregation_test.cpp index 92d8955ab93..7636ab40b59 100644 --- a/vespalib/src/tests/btree/btreeaggregation_test.cpp +++ b/vespalib/src/tests/btree/btreeaggregation_test.cpp @@ -332,6 +332,7 @@ private: void requireThatTreeIteratorAssignWorks(); void requireThatUpdateOfKeyWorks(); void requireThatUpdateOfDataWorks(); + void requireThatFrozenViewProvidesAggregatedValues(); template void @@ -1185,6 +1186,24 @@ Test::requireThatSmallNodesWorks() s.trimHoldLists(g.getFirstUsedGeneration()); } +void +Test::requireThatFrozenViewProvidesAggregatedValues() +{ + MyTree t; + t.insert(20, 102); + t.insert(10, 101); + t.insert(30, 103); + t.insert(40, 104); + auto old_view = t.getFrozenView(); + t.getAllocator().freeze(); + auto new_view = t.getFrozenView(); + auto new_aggregated = new_view.getAggregated(); + EXPECT_EQUAL(new_aggregated.getMin(), 101); + EXPECT_EQUAL(new_aggregated.getMax(), 104); + auto old_aggregated = old_view.getAggregated(); + EXPECT_EQUAL(old_aggregated.getMin(), std::numeric_limits::max()); + EXPECT_EQUAL(old_aggregated.getMax(), std::numeric_limits::min()); +} int Test::Main() @@ -1208,6 +1227,7 @@ Test::Main() requireThatUpdateOfDataWorks(); TEST_DO(requireThatSmallNodesWorks()); TEST_DO(requireThatSmallNodesWorks()); + TEST_DO(requireThatFrozenViewProvidesAggregatedValues()); TEST_DONE(); } diff --git a/vespalib/src/tests/btree/frozenbtree_test.cpp b/vespalib/src/tests/btree/frozenbtree_test.cpp index 597c2ffdc90..1826be19a0a 100644 --- a/vespalib/src/tests/btree/frozenbtree_test.cpp +++ b/vespalib/src/tests/btree/frozenbtree_test.cpp @@ -421,7 +421,9 @@ FrozenBTreeTest::Main() allocTree(); insertRandomValues(*_tree, *_allocator, _randomValues); lookupRandomValues(*_tree, *_allocator, _randomValues); + EXPECT_TRUE(_tree->getFrozenView(*_allocator).empty()); _allocator->freeze(); + EXPECT_FALSE(_tree->getFrozenView(*_allocator).empty()); _allocator->transferHoldLists(_generationHandler->getCurrentGeneration()); lookupFrozenRandomValues(*_tree, *_allocator, _randomValues); traverseTreeIterator(*_tree, diff --git a/vespalib/src/vespa/vespalib/btree/btreeroot.h b/vespalib/src/vespa/vespalib/btree/btreeroot.h index b5759a6a341..471f4273632 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeroot.h +++ b/vespalib/src/vespa/vespalib/btree/btreeroot.h @@ -83,6 +83,12 @@ public: size_t size() const; const NodeAllocatorType &getAllocator() const { return _allocator; } + const AggrT &getAggregated() const { + return _allocator.getAggregated(_frozenRoot); + } + + bool empty() const { return !_frozenRoot.valid(); } + template void foreach_key(FunctionType func) const { _allocator.getNodeStore().foreach_key(_frozenRoot, func); -- cgit v1.2.3 From c7b890535a3009e52e7227b883f2c9a16e59a740 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 25 Apr 2020 18:57:12 +0000 Subject: Let size of thread pool and Q follow number of cores on the machine where it is running. If auto detected num worker threads will use number of #cores * 4, quesize will use #numWorkers * 4. No changes of the default value in this commit. --- .../com/yahoo/config/model/api/ModelContext.java | 6 ++++ .../yahoo/config/model/deploy/TestProperties.java | 20 +++++++++++ .../model/container/ApplicationContainer.java | 18 +++++++++- .../vespa/model/container/ContainerCluster.java | 14 ++++++++ .../vespa/model/container/NodeFlavorTuning.java | 30 ++++++++++++++++- .../model/container/ContainerClusterTest.java | 39 ++++++++++++++++++++++ .../config/server/deploy/ModelContextImpl.java | 18 +++++++++- container-core/abi-spec.json | 2 ++ .../container/handler/ThreadPoolProvider.java | 30 +++++++++++++---- .../resources/configdefinitions/threadpool.def | 8 +++++ .../handler/ThreadPoolProviderTestCase.java | 34 +++++++++++++++++++ .../src/main/java/com/yahoo/vespa/flags/Flags.java | 11 +++++- 12 files changed, 220 insertions(+), 10 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index c406fa80434..86360d482a6 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -71,6 +71,12 @@ public interface ModelContext { // TODO Revisit in May or June 2020 double defaultTermwiseLimit(); + // TODO Revisit in May or June 2020 + double threadPoolSizeFactor(); + + // TODO Revisit in May or June 2020 + double queueSizeFactor(); + // TODO Revisit in May or June 2020 double defaultSoftStartSeconds(); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 99225beba4f..d799af36c3b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -43,6 +43,8 @@ public class TestProperties implements ModelContext.Properties { private double topKProbability = 1.0; private double defaultTermwiseLimit = 1.0; private double softStartSeconds = 0.0; + private double threadPoolSizeFactor = 0.0; + private double queueSizeFactor = 0.0; private Optional endpointCertificateSecrets = Optional.empty(); private AthenzDomain athenzDomain; @@ -64,6 +66,16 @@ public class TestProperties implements ModelContext.Properties { @Override public Optional tlsSecrets() { return endpointCertificateSecrets.map(TlsSecrets::new); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } + @Override + public double threadPoolSizeFactor() { + return threadPoolSizeFactor; + } + + @Override + public double queueSizeFactor() { + return queueSizeFactor; + } + @Override public double defaultSoftStartSeconds() { return softStartSeconds; @@ -86,7 +98,15 @@ public class TestProperties implements ModelContext.Properties { this.softStartSeconds = softStartSeconds; return this; } + public TestProperties setThreadPoolSizeFactor(double threadPoolSizeFactor) { + this.threadPoolSizeFactor = threadPoolSizeFactor; + return this; + } + public TestProperties setQueueSizeFactor(double queueSizeFactor) { + this.queueSizeFactor = queueSizeFactor; + return this; + } public TestProperties setApplicationId(ApplicationId applicationId) { this.applicationId = applicationId; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index e59baf88422..29bb578a67b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.Flavor; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.search.config.QrStartConfig; @@ -15,7 +16,10 @@ import com.yahoo.vespa.model.container.component.Component; * * @author gjoranv */ -public final class ApplicationContainer extends Container implements QrStartConfig.Producer { +public final class ApplicationContainer extends Container implements + QrStartConfig.Producer, + ThreadpoolConfig.Producer +{ private static final String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage"; @@ -73,4 +77,16 @@ public final class ApplicationContainer extends Container implements QrStartConf return (parent instanceof ContainerCluster) && (((ContainerCluster)parent).getDocproc() != null); } + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + if (! (parent instanceof ContainerCluster)) return; + if ((getHostResource() == null) || getHostResource().getFlavor().isEmpty()) return; + ContainerCluster containerCluster = (ContainerCluster) parent; + if (containerCluster.getThreadPoolSizeFactor() <= 0.0) return; + + NodeFlavorTuning flavorTuning = new NodeFlavorTuning(getHostResource().getFlavor().get()) + .setThreadPoolSizeFactor(containerCluster.getThreadPoolSizeFactor()) + .setQueueSizeFactor(containerCluster.getQueueSizeFactor()); + flavorTuning.getConfig(builder); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 833f6688fbc..b35b7562704 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -160,12 +160,18 @@ public abstract class ContainerCluster private String jvmGCOptions = null; private String environmentVars = null; + private final double threadPoolSizeFactor; + private final double queueSizeFactor; + public ContainerCluster(AbstractConfigProducer parent, String subId, String name, DeployState deployState) { super(parent, subId); this.name = name; this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); + this.threadPoolSizeFactor = deployState.getProperties().threadPoolSizeFactor(); + this.queueSizeFactor = deployState.getProperties().queueSizeFactor(); + componentGroup = new ComponentGroup<>(this, "component"); addComponent(new StatisticsComponent()); @@ -186,6 +192,14 @@ public abstract class ContainerCluster addJaxProviders(); } + public double getThreadPoolSizeFactor() { + return threadPoolSizeFactor; + } + + public double getQueueSizeFactor() { + return queueSizeFactor; + } + public void setZone(Zone zone) { this.zone = zone; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java index 67938b36fd9..f9b50d0e641 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.container; import com.yahoo.config.provision.Flavor; +import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; /** @@ -9,10 +10,26 @@ import com.yahoo.search.config.QrStartConfig; * * @author balder */ -public class NodeFlavorTuning implements QrStartConfig.Producer { +public class NodeFlavorTuning implements + QrStartConfig.Producer, + ThreadpoolConfig.Producer +{ private final Flavor flavor; + public NodeFlavorTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) { + this.threadPoolSizeFactor = threadPoolSizeFactor; + return this; + } + + public NodeFlavorTuning setQueueSizeFactor(double queueSizeFactor) { + this.queueSizeFactor = queueSizeFactor; + return this; + } + + private double threadPoolSizeFactor = 8.0; + private double queueSizeFactor = 8.0; + NodeFlavorTuning(Flavor flavor) { this.flavor = flavor; } @@ -22,4 +39,15 @@ public class NodeFlavorTuning implements QrStartConfig.Producer { builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(flavor.getMinCpuCores()))); } + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + // Controls max number of concurrent requests per container + int workerThreads = Math.max(2, (int)Math.ceil(flavor.getMinCpuCores() * threadPoolSizeFactor)); + builder.maxthreads(workerThreads); + + // This controls your burst handling capability. + // 0 => No extra burst handling beyond you max concurrent requests (maxthreads). + // N => N times max concurrent requests as a buffer for handling bursts + builder.queueSize((int)(workerThreads * queueSizeFactor)); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 33cf0635349..0f94df80421 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -9,9 +9,11 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.Host; @@ -164,6 +166,8 @@ public class ContainerClusterTest { cluster.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(10, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); + assertEquals(0, threadpoolConfig.softStartSeconds(), 0); } @Test @@ -213,9 +217,43 @@ public class ContainerClusterTest { cluster.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(500, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); assertEquals(300.0, threadpoolConfig.softStartSeconds(), 0.0); } + @Test + public void requireThatPoolAndQueueCanNotBeControlledByPropertiesWhenNoFlavor() { + DeployState state = new DeployState.Builder().properties(new TestProperties() + .setThreadPoolSizeFactor(8.5) + .setQueueSizeFactor(13.3)) + .build(); + MockRoot root = new MockRoot("foo", state); + ApplicationContainerCluster cluster = createContainerCluster(root, false); + addContainer(root.deployLogger(), cluster, "c1", "host-c1"); + + ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); + cluster.getConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + assertEquals(500, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); + assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); + } + + @Test + public void requireThatPoolAndQueueCanBeControlledByPropertiesAndFlavor() { + FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder().name("my_flavor").minCpuCores(3); + NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder))) + .setThreadPoolSizeFactor(13.3) + .setQueueSizeFactor(17.5); + + ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); + nodeFlavorTuning.getConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + assertEquals(40, threadpoolConfig.maxthreads()); + assertEquals(700, threadpoolConfig.queueSize()); + assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); + } + @Test public void requireThatDefaultThreadPoolConfigIsSane() { MockRoot root = new MockRoot("foo"); @@ -226,6 +264,7 @@ public class ContainerClusterTest { cluster.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(500, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 2773659559b..0f141dee5b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -150,6 +150,8 @@ public class ModelContextImpl implements ModelContext { private final Optional endpointCertificateSecrets; private final double defaultTermwiseLimit; private final double defaultSoftStartSeconds; + private final double threadPoolSizeFactor; + private final double queueSizefactor; private final Optional athenzDomain; public Properties(ApplicationId applicationId, @@ -186,6 +188,10 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); defaultTopKprobability = Flags.DEFAULT_TOP_K_PROBABILITY.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + threadPoolSizeFactor = Flags.DEFAULT_THREADPOOL_SIZE_FACTOR.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.athenzDomain = athenzDomain; } @@ -238,8 +244,18 @@ public class ModelContextImpl implements ModelContext { @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } + @Override + public double threadPoolSizeFactor() { + return threadPoolSizeFactor; + } + + @Override + public double queueSizeFactor() { + return queueSizefactor; + } + public double defaultSoftStartSeconds() { - return 0; + return defaultSoftStartSeconds; } @Override diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 6d683c53984..82d34d88b99 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -171,6 +171,7 @@ "public void ()", "public void (com.yahoo.container.handler.ThreadpoolConfig)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxthreads(int)", + "public com.yahoo.container.handler.ThreadpoolConfig$Builder queueSize(int)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxThreadExecutionTimeSeconds(int)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder softStartSeconds(double)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", @@ -210,6 +211,7 @@ "public static java.lang.String getDefVersion()", "public void (com.yahoo.container.handler.ThreadpoolConfig$Builder)", "public int maxthreads()", + "public int queueSize()", "public int maxThreadExecutionTimeSeconds()", "public double softStartSeconds()" ], diff --git a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java index b427a58c9b7..0e786cfbc8f 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java @@ -10,6 +10,7 @@ import com.yahoo.container.protect.ProcessTerminator; import com.yahoo.jdisc.Metric; import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; @@ -33,17 +34,30 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider createQ(int queueSize, int maxThreads) { + return (queueSize == 0) + ? new SynchronousQueue<>(false) + : (queueSize < 0) + ? new ArrayBlockingQueue<>(maxThreads*4) + : new ArrayBlockingQueue<>(queueSize); + } + + private static int computeThreadPoolSize(int maxNumThreads) { + return (maxNumThreads <= 0) + ? Runtime.getRuntime().availableProcessors() * 4 + : maxNumThreads; + } @Inject public ThreadPoolProvider(ThreadpoolConfig threadpoolConfig, Metric metric) { this(threadpoolConfig, metric, new ProcessTerminator()); } public ThreadPoolProvider(ThreadpoolConfig threadpoolConfig, Metric metric, ProcessTerminator processTerminator) { + int maxNumThreads = computeThreadPoolSize(threadpoolConfig.maxthreads()); WorkerCompletionTimingThreadPoolExecutor executor = - new WorkerCompletionTimingThreadPoolExecutor(threadpoolConfig.maxthreads(), - threadpoolConfig.maxthreads(), + new WorkerCompletionTimingThreadPoolExecutor(maxNumThreads, maxNumThreads, 0L, TimeUnit.SECONDS, - new SynchronousQueue<>(false), + createQ(threadpoolConfig.queueSize(), maxNumThreads), ThreadFactoryFactory.getThreadFactory("threadpool"), metric); // Prestart needed, if not all threads will be created by the fist N tasks and hence they might also @@ -87,8 +101,9 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider didItRun = new Receiver<>(); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 76206996207..85d841e4bb0 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -150,7 +150,16 @@ public class Flags { "Default number of seconds that a soft start shall use", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - + public static final UnboundDoubleFlag DEFAULT_THREADPOOL_SIZE_FACTOR = defineDoubleFlag( + "default-threadpool-size-factor", 0.0, + "Default multiplication factor when computing maxthreads for main container threadpool based on available cores", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag DEFAULT_QUEUE_SIZE_FACTOR = defineDoubleFlag( + "default-queue-size-factor", 0.0, + "Default multiplication factor when computing queuesize for burst handling", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag DEFAULT_TOP_K_PROBABILITY = defineDoubleFlag( "default-top-k-probability", 1.0, "Default probability that you will get the globally top K documents when merging many partitions.", -- cgit v1.2.3 From d4bf2e41fab0adce629308de75b3e1763dfadccb Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Sat, 25 Apr 2020 22:51:49 +0200 Subject: Reduce code duplication. Rand48 class is in vespalib. --- .../apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp | 26 ++++++------- searchlib/src/apps/tests/btreestress_test.cpp | 8 ++-- .../src/apps/tests/memoryindexstress_test.cpp | 8 ++-- .../attributefilewriter_test.cpp | 4 +- .../enumeratedsave/enumeratedsave_test.cpp | 4 +- .../multi_value_mapping_test.cpp | 4 +- .../tests/attribute/postinglist/postinglist.cpp | 4 +- .../src/tests/common/bitvector/bitvector_test.cpp | 6 +-- .../diskindex/fieldwriter/fieldwriter_test.cpp | 4 +- .../tests/diskindex/pagedict4/pagedict4test.cpp | 12 +++--- .../src/tests/features/beta/beta_features.cpp | 2 +- searchlib/src/tests/features/prod_features.cpp | 6 +-- .../src/tests/postinglistbm/posting_list_test.cpp | 4 +- .../src/tests/postinglistbm/postinglistbm.cpp | 4 +- .../src/tests/postinglistbm/stress_runner.cpp | 8 ++-- searchlib/src/tests/postinglistbm/stress_runner.h | 9 ++--- .../sparse_vector_benchmark_test.cpp | 2 +- .../tests/transactionlogstress/translogstress.cpp | 8 ++-- .../tests/util/bufferwriter/bufferwriter_test.cpp | 4 +- .../src/vespa/searchlib/features/randomfeature.h | 6 +-- .../src/vespa/searchlib/test/fakedata/fakeword.cpp | 16 ++++---- .../src/vespa/searchlib/test/fakedata/fakeword.h | 12 +++--- .../vespa/searchlib/test/fakedata/fakewordset.cpp | 4 +- .../vespa/searchlib/test/fakedata/fakewordset.h | 6 +-- searchlib/src/vespa/searchlib/util/rand48.h | 43 ---------------------- searchlib/src/vespa/searchlib/util/random_normal.h | 4 +- .../src/vespa/searchlib/util/randomgenerator.h | 4 +- vespalib/src/tests/btree/btree_test.cpp | 2 +- vespalib/src/tests/btree/btreeaggregation_test.cpp | 2 +- vespalib/src/tests/btree/frozenbtree_test.cpp | 2 +- vespalib/src/vespa/vespalib/util/rand48.h | 10 +++-- 31 files changed, 97 insertions(+), 141 deletions(-) delete mode 100644 searchlib/src/vespa/searchlib/util/rand48.h diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index 2f54cefac29..080d98734bd 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include +#include #include #include #include @@ -98,10 +98,10 @@ shafile(const string &baseDir, class StringGenerator { - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; public: - StringGenerator(search::Rand48 &rnd); + StringGenerator(vespalib::Rand48 &rnd); void rand_string(string &res, uint32_t minLen, uint32_t maxLen); @@ -114,7 +114,7 @@ public: }; -StringGenerator::StringGenerator(search::Rand48 &rnd) +StringGenerator::StringGenerator(vespalib::Rand48 &rnd) : _rnd(rnd) { } @@ -283,7 +283,7 @@ PrefixTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t id) class RandTextFieldGenerator : public FieldGenerator { - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; uint32_t _numWords; StringArray _strings; uint32_t _minFill; @@ -291,7 +291,7 @@ class RandTextFieldGenerator : public FieldGenerator public: RandTextFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t numWords, uint32_t minFill, uint32_t maxFill); @@ -302,7 +302,7 @@ public: RandTextFieldGenerator::RandTextFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t numWords, uint32_t minFill, uint32_t randFill) @@ -354,7 +354,7 @@ class ModTextFieldGenerator : public FieldGenerator public: ModTextFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const std::vector &mods); virtual ~ModTextFieldGenerator(); virtual void generateValue(vespalib::asciistream &doc, uint32_t id) override; @@ -362,7 +362,7 @@ public: ModTextFieldGenerator::ModTextFieldGenerator(const string &name, - [[maybe_unused]] search::Rand48 &rnd, + [[maybe_unused]] vespalib::Rand48 &rnd, const std::vector &mods) : FieldGenerator(name), _mods(mods) @@ -419,13 +419,13 @@ IdTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t id) class RandIntFieldGenerator : public FieldGenerator { - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; uint32_t _low; uint32_t _count; public: RandIntFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t low, uint32_t count); virtual ~RandIntFieldGenerator(); @@ -436,7 +436,7 @@ public: RandIntFieldGenerator::RandIntFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t low, uint32_t count) : FieldGenerator(name), @@ -617,7 +617,7 @@ class GenTestDocsApp : public SubApp int _optIndex; std::vector _fields; std::vector _mods; - search::Rand48 _rnd; + vespalib::Rand48 _rnd; string _outFile; bool _headers; bool _json; diff --git a/searchlib/src/apps/tests/btreestress_test.cpp b/searchlib/src/apps/tests/btreestress_test.cpp index e6ef0740fc3..a39416c2227 100644 --- a/searchlib/src/apps/tests/btreestress_test.cpp +++ b/searchlib/src/apps/tests/btreestress_test.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include @@ -37,7 +37,7 @@ struct Fixture MyTreeIterator _writeItr; vespalib::ThreadStackExecutor _writer; // 1 write thread vespalib::ThreadStackExecutor _readers; // multiple reader threads - search::Rand48 _rnd; + vespalib::Rand48 _rnd; uint32_t _keyLimit; std::atomic _readSeed; std::atomic _doneWriteWork; @@ -135,7 +135,7 @@ Fixture::remove(uint32_t key) void Fixture::readWork(uint32_t cnt) { - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(++_readSeed); uint32_t i; for (i = 0; i < cnt && _stopRead.load() == 0; ++i) { @@ -159,7 +159,7 @@ Fixture::readWork() void Fixture::writeWork(uint32_t cnt) { - search::Rand48 &rnd(_rnd); + vespalib::Rand48 &rnd(_rnd); for (uint32_t i = 0; i < cnt; ++i) { uint32_t key = rnd.lrand48() % _keyLimit; if ((rnd.lrand48() & 1) == 0) { diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 6bbd93bda84..041335b7ed1 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include @@ -201,7 +201,7 @@ struct Fixture { uint32_t _readThreads; vespalib::ThreadStackExecutor _writer; // 1 write thread vespalib::ThreadStackExecutor _readers; // multiple reader threads - search::Rand48 _rnd; + vespalib::Rand48 _rnd; uint32_t _keyLimit; std::atomic _readSeed; std::atomic _doneWriteWork; @@ -285,7 +285,7 @@ Fixture::~Fixture() void Fixture::readWork(uint32_t cnt) { - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(++_readSeed); uint32_t i; uint32_t emptyCount = 0; @@ -345,7 +345,7 @@ Fixture::readWork() void Fixture::writeWork(uint32_t cnt) { - search::Rand48 &rnd(_rnd); + vespalib::Rand48 &rnd(_rnd); for (uint32_t i = 0; i < cnt; ++i) { uint32_t key = rnd.lrand48() % _keyLimit; if ((rnd.lrand48() & 1) == 0) { diff --git a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp index 8d13987bb4f..eeabbf23c27 100644 --- a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp +++ b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include @@ -77,7 +77,7 @@ TEST_F("Test that buffer writer passes on written data", Fixture) const size_t writerBufferSize = AttributeFileBufferWriter::BUFFER_SIZE; EXPECT_GREATER(mysize * sizeof(int), writerBufferSize); a.reserve(mysize); - search::Rand48 rnd; + vespalib::Rand48 rnd; for (uint32_t i = 0; i < mysize; ++i) { a.emplace_back(rnd.lrand48()); } diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index 41313fc7c53..eeaf85472c3 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -227,7 +227,7 @@ void EnumeratedSaveTest::populate(IntegerAttribute &v, unsigned seed, BasicType bt) { - search::Rand48 rnd; + vespalib::Rand48 rnd; IntegerAttribute::largeint_t mask(std::numeric_limits ::max()); switch (bt.type()) { @@ -284,7 +284,7 @@ EnumeratedSaveTest::populate(FloatingPointAttribute &v, unsigned seed, BasicType bt) { (void) bt; - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(seed); int weight = 1; for(size_t i(0), m(v.getNumDocs()); i < m; i++) { diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp index 2f74e9d31f6..a936ba124ee 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -151,7 +151,7 @@ using IntMappingTest = MappingTestBase; class CompactionIntMappingTest : public MappingTestBase { - search::Rand48 _rnd; + vespalib::Rand48 _rnd; std::map> _refMapping; public: CompactionIntMappingTest() diff --git a/searchlib/src/tests/attribute/postinglist/postinglist.cpp b/searchlib/src/tests/attribute/postinglist/postinglist.cpp index 78b93e2b78b..37d76e4a9e8 100644 --- a/searchlib/src/tests/attribute/postinglist/postinglist.cpp +++ b/searchlib/src/tests/attribute/postinglist/postinglist.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include @@ -143,7 +143,7 @@ private: PostingList *_intPostings; STLValueTree *_stlTree; - Rand48 _randomGenerator; + vespalib::Rand48 _randomGenerator; uint32_t _generation; void diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index 409cc9f2725..db29ab9881f 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include using namespace search; @@ -81,7 +81,7 @@ myCountInterval(const BitVector &bv, uint32_t low, uint32_t high) } void -scan(uint32_t count, uint32_t offset, uint32_t size, Rand48 &rnd) +scan(uint32_t count, uint32_t offset, uint32_t size, vespalib::Rand48 &rnd) { std::vector lids; lids.reserve(count); @@ -110,7 +110,7 @@ scan(uint32_t count, uint32_t offset, uint32_t size, Rand48 &rnd) void scanWithOffset(uint32_t offset) { - Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(32); scan(10, offset, 1000000, rnd); diff --git a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp index ef55ea60c2b..15a53497db9 100644 --- a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp +++ b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include -#include +#include #include #include #include @@ -99,7 +99,7 @@ private: FakeWordSet _wordSet; FakeWordSet _wordSet2; public: - search::Rand48 _rnd; + vespalib::Rand48 _rnd; private: void Usage(); diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp index e914fe7c559..6cc2f318f4b 100644 --- a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include -#include +#include #include #include #include @@ -54,7 +54,7 @@ using RandReader = search::diskindex::test::PageDict4MemRandReader; class PageDict4TestApp : public FastOS_Application { public: - search::Rand48 _rnd; + vespalib::Rand48 _rnd; bool _stress; bool _emptyWord; bool _firstWordForcedCommon; @@ -206,7 +206,7 @@ deDup(std::vector &v) static WordIndexCounts -makeIndex(search::Rand48 &rnd, bool forceCommon) +makeIndex(vespalib::Rand48 &rnd, bool forceCommon) { uint64_t bitLength = 10; uint32_t numDocs = 1; @@ -219,7 +219,7 @@ makeIndex(search::Rand48 &rnd, bool forceCommon) void -makeIndexes(search::Rand48 &rnd, +makeIndexes(vespalib::Rand48 &rnd, WordIndexCounts &counts, bool forceCommon) { @@ -229,7 +229,7 @@ makeIndexes(search::Rand48 &rnd, static void makeWords(std::vector &v, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t numWordIds, uint32_t tupleCount, bool emptyWord, @@ -355,7 +355,7 @@ checkCounts(const std::string &word, void testWords(const std::string &logname, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint64_t numWordIds, uint32_t tupleCount, uint32_t chunkSize, diff --git a/searchlib/src/tests/features/beta/beta_features.cpp b/searchlib/src/tests/features/beta/beta_features.cpp index 617bf2fb073..89a72fab6fc 100644 --- a/searchlib/src/tests/features/beta/beta_features.cpp +++ b/searchlib/src/tests/features/beta/beta_features.cpp @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 1c07c81bc2f..3f71c9f85ea 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -40,7 +40,7 @@ #include #include #include -#include +#include #include #include #include @@ -1747,7 +1747,7 @@ Test::testRandom() FtFeatureTest ft(_factory, "random"); ft.getIndexEnv().getProperties().add("random.seed", "100"); ASSERT_TRUE(ft.setup()); - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(100); for (uint32_t i = 0; i < 5; ++i) { feature_t exp = static_cast(rnd.lrand48()) / static_cast(0x80000000u); @@ -1770,7 +1770,7 @@ Test::testRandom() FtFeatureTest ft(_factory, "random.match"); ft.getQueryEnv().getProperties().add("random.match.seed", "100"); ASSERT_TRUE(ft.setup()); - search::Rand48 rnd; + vespalib::Rand48 rnd; for (uint32_t i = 1; i <= 5; ++i) { rnd.srand48(100 + i); // seed + lid feature_t exp = static_cast(rnd.lrand48()) / static_cast(0x80000000u); diff --git a/searchlib/src/tests/postinglistbm/posting_list_test.cpp b/searchlib/src/tests/postinglistbm/posting_list_test.cpp index a7d810cb1a1..845201772f8 100644 --- a/searchlib/src/tests/postinglistbm/posting_list_test.cpp +++ b/searchlib/src/tests/postinglistbm/posting_list_test.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include @@ -66,7 +66,7 @@ struct PostingListTest : public ::testing::Test { FakeWordUP word3; FakeWordUP word4; FakeWordUP word5; - search::Rand48 rnd; + vespalib::Rand48 rnd; PostingListTest() : num_docs(36000), diff --git a/searchlib/src/tests/postinglistbm/postinglistbm.cpp b/searchlib/src/tests/postinglistbm/postinglistbm.cpp index 16b8e9cd7f5..0ac3f15e015 100644 --- a/searchlib/src/tests/postinglistbm/postinglistbm.cpp +++ b/searchlib/src/tests/postinglistbm/postinglistbm.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include @@ -42,7 +42,7 @@ private: bool _unpack; public: - search::Rand48 _rnd; + vespalib::Rand48 _rnd; public: PostingListBM(); diff --git a/searchlib/src/tests/postinglistbm/stress_runner.cpp b/searchlib/src/tests/postinglistbm/stress_runner.cpp index 9c78515a03f..fd12e9b5aa9 100644 --- a/searchlib/src/tests/postinglistbm/stress_runner.cpp +++ b/searchlib/src/tests/postinglistbm/stress_runner.cpp @@ -33,7 +33,7 @@ private: StressMaster &operator=(const StressMaster &); - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; uint32_t _numDocs; std::vector _postingTypes; StressRunner::OperatorType _operatorType; @@ -62,7 +62,7 @@ private: std::vector _tasks; public: - StressMaster(search::Rand48 &rnd, + StressMaster(vespalib::Rand48 &rnd, FakeWordSet &wordSet, const std::vector &postingType, StressRunner::OperatorType operatorType, @@ -129,7 +129,7 @@ public: }; -StressMaster::StressMaster(search::Rand48 &rnd, +StressMaster::StressMaster(vespalib::Rand48 &rnd, FakeWordSet &wordSet, const std::vector &postingTypes, StressRunner::OperatorType operatorType, @@ -420,7 +420,7 @@ OrStressWorker::run_task(const FakePosting& f1, const FakePosting& f2, uint32_t } void -StressRunner::run(search::Rand48 &rnd, +StressRunner::run(vespalib::Rand48 &rnd, FakeWordSet &wordSet, const std::vector &postingTypes, OperatorType operatorType, diff --git a/searchlib/src/tests/postinglistbm/stress_runner.h b/searchlib/src/tests/postinglistbm/stress_runner.h index d4974bc969e..249645f70aa 100644 --- a/searchlib/src/tests/postinglistbm/stress_runner.h +++ b/searchlib/src/tests/postinglistbm/stress_runner.h @@ -4,12 +4,9 @@ #include #include -namespace search { -class Rand48; +namespace vespalib { class Rand48; } -namespace fakedata { class FakeWordSet; } - -} +namespace search::fakedata { class FakeWordSet; } namespace postinglistbm { @@ -21,7 +18,7 @@ public: Or }; - static void run(search::Rand48 &rnd, + static void run(vespalib::Rand48 &rnd, search::fakedata::FakeWordSet &wordSet, const std::vector &postingTypes, OperatorType operatorType, diff --git a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp index a8aab279a98..1cf39183206 100644 --- a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp +++ b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index 74e48081e17..81a3006dbff 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -37,7 +37,7 @@ using Visitor = TransLogClient::Visitor; class BufferGenerator { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; uint32_t _minStrLen; uint32_t _maxStrLen; @@ -71,7 +71,7 @@ BufferGenerator::getRandomBuffer() class EntryGenerator { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; long _baseSeed; BufferGenerator _bufferGenerator; const std::vector * _buffers; @@ -93,7 +93,7 @@ public: }; SerialNum getRandomSerialNum(SerialNum begin, SerialNum end); Packet::Entry getRandomEntry(SerialNum num); - Rand48 & getRnd() { return _rnd; } + vespalib::Rand48 & getRnd() { return _rnd; } void setBuffers(const std::vector & buffers) { _buffers = &buffers; } diff --git a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp index 33afa67e660..8787712663d 100644 --- a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp +++ b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include namespace search { @@ -130,7 +130,7 @@ TEST("Test that bufferwriter passes on written data") const size_t drainerBufferSize = DrainingBufferWriter::BUFFER_SIZE; EXPECT_GREATER(mysize * sizeof(int), drainerBufferSize); a.reserve(mysize); - search::Rand48 rnd; + vespalib::Rand48 rnd; for (uint32_t i = 0; i < mysize; ++i) { a.emplace_back(rnd.lrand48()); } diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.h b/searchlib/src/vespa/searchlib/features/randomfeature.h index 8c94b71071f..3a93e77d4d0 100644 --- a/searchlib/src/vespa/searchlib/features/randomfeature.h +++ b/searchlib/src/vespa/searchlib/features/randomfeature.h @@ -3,7 +3,7 @@ #pragma once #include -#include +#include namespace search::features { @@ -12,8 +12,8 @@ namespace search::features { **/ class RandomExecutor : public search::fef::FeatureExecutor { private: - Rand48 _rnd; // seeded once per query - Rand48 _matchRnd; // seeded once per match + vespalib::Rand48 _rnd; // seeded once per query + vespalib::Rand48 _matchRnd; // seeded once per match uint64_t _matchSeed; public: diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp index 19080acaad3..4681861dd73 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp @@ -31,7 +31,7 @@ namespace fakedata static void fillbitset(search::BitVector *bitvector, unsigned int size, - search::Rand48 &rnd) + vespalib::Rand48 &rnd) { unsigned int range; unsigned int idx; @@ -70,7 +70,7 @@ static void fillcorrelatedbitset(search::BitVector &bitvector, unsigned int size, const FakeWord &otherword, - search::Rand48 &rnd) + vespalib::Rand48 &rnd) { const FakeWord::DocWordFeatureList &opostings = otherword._postings; @@ -145,7 +145,7 @@ FakeWord::FakeWord(uint32_t docIdLimit, for (uint32_t docId : docIds) { bitmap->setBit(docId); } - search::Rand48 rnd; + vespalib::Rand48 rnd; fakeup(*bitmap, rnd, _postings, _wordPosFeatures); } @@ -153,7 +153,7 @@ FakeWord::FakeWord(uint32_t docIdLimit, uint32_t wordDocs, uint32_t tempWordDocs, const std::string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex) : _postings(), @@ -181,7 +181,7 @@ FakeWord::FakeWord(uint32_t docIdLimit, const std::string &name, const FakeWord &otherWord, size_t overlapDocs, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex) : _postings(), @@ -211,7 +211,7 @@ FakeWord::~FakeWord() void FakeWord::fakeup(search::BitVector &bitmap, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, DocWordFeatureList &postings, DocWordPosFeatureList &wordPosFeatures) { @@ -301,7 +301,7 @@ FakeWord::fakeup(search::BitVector &bitmap, void -FakeWord::fakeupTemps(search::Rand48 &rnd, +FakeWord::fakeupTemps(vespalib::Rand48 &rnd, uint32_t docIdLimit, uint32_t tempWordDocs) { @@ -315,7 +315,7 @@ FakeWord::fakeupTemps(search::Rand48 &rnd, } void -FakeWord::setupRandomizer(search::Rand48 &rnd) +FakeWord::setupRandomizer(vespalib::Rand48 &rnd) { typedef DocWordFeatureList DWFL; Randomizer randomAdd; diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h index 30f960c2d4c..ffe9e2e54d0 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include +#include #include #include #include @@ -156,16 +156,16 @@ public: void fakeup(search::BitVector &bitmap, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, DocWordFeatureList &postings, DocWordPosFeatureList &wordPosFeatures); void - fakeupTemps(search::Rand48 &rnd, + fakeupTemps(vespalib::Rand48 &rnd, uint32_t docIdLimit, uint32_t tempWordDocs); - void setupRandomizer(search::Rand48 &rnd); + void setupRandomizer(vespalib::Rand48 &rnd); const DocWordFeature & getDocWordFeature(const Randomizer &r) const @@ -220,7 +220,7 @@ public: uint32_t wordDocs, uint32_t tempWordDocs, const std::string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex); @@ -230,7 +230,7 @@ public: const std::string &name, const FakeWord &otherWord, size_t overlapDocs, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp index ba691e507fa..6bb39598fab 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp @@ -77,7 +77,7 @@ FakeWordSet::setupParams(bool hasElements, } void -FakeWordSet::setupWords(search::Rand48 &rnd, +FakeWordSet::setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t numWordsPerWordClass) @@ -86,7 +86,7 @@ FakeWordSet::setupWords(search::Rand48 &rnd, } void -FakeWordSet::setupWords(search::Rand48 &rnd, +FakeWordSet::setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t mediumDocFreq, diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h index 3dcf47cf8c8..de7efef1517 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h @@ -6,7 +6,7 @@ #include #include -namespace search { class Rand48; } +namespace vespalib { class Rand48; } namespace search::fakedata { @@ -46,12 +46,12 @@ public: void setupParams(bool hasElements, bool hasElementWeights); - void setupWords(search::Rand48 &rnd, + void setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t numWordsPerWordClass); - void setupWords(search::Rand48 &rnd, + void setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t mediumDocFreq, diff --git a/searchlib/src/vespa/searchlib/util/rand48.h b/searchlib/src/vespa/searchlib/util/rand48.h deleted file mode 100644 index 8ef529493c1..00000000000 --- a/searchlib/src/vespa/searchlib/util/rand48.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include -#include - -namespace search { - -/* - * Simple random generator based on lrand48() spec. - */ -class Rand48 -{ -private: - uint64_t _state; -public: - void - srand48(long seed) - { - _state = ((static_cast(seed & 0xffffffffu)) << 16) + 0x330e; - } - - Rand48() - : _state(0) - { - srand48(0x1234abcd); - }; - void iterate() { - _state = (UINT64_C(0x5DEECE66D) * _state + 0xb) & - UINT64_C(0xFFFFFFFFFFFF); - } - /* - * Return value from 0 to 2^31 - 1 - */ - long lrand48() { - iterate(); - return static_cast(_state >> 17); - } -}; - -} // namespace search - diff --git a/searchlib/src/vespa/searchlib/util/random_normal.h b/searchlib/src/vespa/searchlib/util/random_normal.h index 74596066312..c74935de02f 100644 --- a/searchlib/src/vespa/searchlib/util/random_normal.h +++ b/searchlib/src/vespa/searchlib/util/random_normal.h @@ -2,7 +2,7 @@ #pragma once -#include +#include #include namespace search { @@ -14,7 +14,7 @@ namespace search { class RandomNormal { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; double _mean; double _stddev; diff --git a/searchlib/src/vespa/searchlib/util/randomgenerator.h b/searchlib/src/vespa/searchlib/util/randomgenerator.h index 40305a1d02d..ac23b387740 100644 --- a/searchlib/src/vespa/searchlib/util/randomgenerator.h +++ b/searchlib/src/vespa/searchlib/util/randomgenerator.h @@ -2,7 +2,7 @@ #pragma once -#include +#include #include #include #include @@ -11,7 +11,7 @@ namespace search { class RandomGenerator { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; public: RandomGenerator() : _rnd() {} diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index 4090283c10f..042a1f69bb2 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -673,7 +673,7 @@ void generateData(std::vector & data, size_t numEntries) { data.reserve(numEntries); - Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(10); for (size_t i = 0; i < numEntries; ++i) { int num = rnd.lrand48() % 10000000; diff --git a/vespalib/src/tests/btree/btreeaggregation_test.cpp b/vespalib/src/tests/btree/btreeaggregation_test.cpp index 92d8955ab93..b64bbb105c5 100644 --- a/vespalib/src/tests/btree/btreeaggregation_test.cpp +++ b/vespalib/src/tests/btree/btreeaggregation_test.cpp @@ -709,7 +709,7 @@ void generateData(std::vector & data, size_t numEntries) { data.reserve(numEntries); - Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(10); for (size_t i = 0; i < numEntries; ++i) { int num = rnd.lrand48() % 10000000; diff --git a/vespalib/src/tests/btree/frozenbtree_test.cpp b/vespalib/src/tests/btree/frozenbtree_test.cpp index 597c2ffdc90..42410d824e8 100644 --- a/vespalib/src/tests/btree/frozenbtree_test.cpp +++ b/vespalib/src/tests/btree/frozenbtree_test.cpp @@ -47,7 +47,7 @@ private: NodeAllocator *_allocator; Tree *_tree; - Rand48 _randomGenerator; + vespalib::Rand48 _randomGenerator; void allocTree(); void freeTree(bool verbose); diff --git a/vespalib/src/vespa/vespalib/util/rand48.h b/vespalib/src/vespa/vespalib/util/rand48.h index 441f8e6e10f..3ebfe00ba35 100644 --- a/vespalib/src/vespa/vespalib/util/rand48.h +++ b/vespalib/src/vespa/vespalib/util/rand48.h @@ -2,8 +2,10 @@ #pragma once +#include +#include -namespace search { +namespace vespalib { /* * Simple random generator based on lrand48() spec. @@ -19,19 +21,19 @@ public: _state = ((static_cast(seed & 0xffffffffu)) << 16) + 0x330e; } - Rand48(void) + Rand48() : _state(0) { srand48(0x1234abcd); }; - void iterate(void) { + void iterate() { _state = (UINT64_C(0x5DEECE66D) * _state + 0xb) & UINT64_C(0xFFFFFFFFFFFF); } /* * Return value from 0 to 2^31 - 1 */ - long lrand48(void) { + long lrand48() { iterate(); return static_cast(_state >> 17); } -- cgit v1.2.3 From 2c0e3e3c68772d06b2a842c1c5403b382763deec Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Sat, 25 Apr 2020 23:42:21 +0200 Subject: Move BufferWriter back to searchlib. --- .../proton/documentmetastore/documentmetastore.cpp | 2 +- .../documentmetastore/documentmetastoresaver.cpp | 2 +- .../enumeratedsave/enumeratedsave_test.cpp | 2 +- .../save_target/attribute_save_target_test.cpp | 2 +- .../tensorattribute/tensorattribute_test.cpp | 2 +- .../tensor/hnsw_saver/hnsw_save_load_test.cpp | 2 +- .../tests/util/bufferwriter/bufferwriter_test.cpp | 2 +- .../attribute/attributefilebufferwriter.h | 2 +- .../searchlib/attribute/enum_store_dictionary.cpp | 2 +- .../searchlib/attribute/enumattributesaver.cpp | 2 +- .../src/vespa/searchlib/attribute/enumstore.hpp | 2 +- .../attribute/multinumericattributesaver.cpp | 2 +- .../searchlib/attribute/multistringattribute.hpp | 2 +- .../attribute/multivalueattributesaverutils.cpp | 2 +- .../attribute/multivalueattributesaverutils.h | 2 +- .../attribute/reference_attribute_saver.cpp | 2 +- .../attribute/singleenumattributesaver.cpp | 2 +- .../searchlib/attribute/singlestringattribute.hpp | 2 +- .../tensor/dense_tensor_attribute_saver.cpp | 2 +- .../tensor/generic_tensor_attribute_saver.cpp | 2 +- .../vespa/searchlib/tensor/hnsw_index_saver.cpp | 2 +- searchlib/src/vespa/searchlib/util/CMakeLists.txt | 1 + .../src/vespa/searchlib/util/bufferwriter.cpp | 36 ++++++++++++++ searchlib/src/vespa/searchlib/util/bufferwriter.h | 56 ++++++++++++++++++++++ .../vespa/searchlib/util/drainingbufferwriter.h | 2 +- .../src/vespa/vespalib/datastore/unique_store.hpp | 1 - vespalib/src/vespa/vespalib/util/CMakeLists.txt | 1 - vespalib/src/vespa/vespalib/util/bufferwriter.cpp | 36 -------------- vespalib/src/vespa/vespalib/util/bufferwriter.h | 56 ---------------------- 29 files changed, 115 insertions(+), 116 deletions(-) create mode 100644 searchlib/src/vespa/searchlib/util/bufferwriter.cpp create mode 100644 searchlib/src/vespa/searchlib/util/bufferwriter.h delete mode 100644 vespalib/src/vespa/vespalib/util/bufferwriter.cpp delete mode 100644 vespalib/src/vespa/vespalib/util/bufferwriter.h diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 3e9ef787f74..c36ee8e474a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include "document_meta_store_versions.h" diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp index 2a3fd14c6d2..7d85c01ee74 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentmetastoresaver.h" -#include +#include #include "document_meta_store_versions.h" #include diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index eeaf85472c3..d8761f69d71 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include diff --git a/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp index c746a0aa120..103551dd93a 100644 --- a/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp +++ b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 39a7e53ca8c..56b8460fbb6 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include LOG_SETUP("tensorattribute_test"); diff --git a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp index b9e27d413f3..7aa78fbe06b 100644 --- a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp index 8787712663d..04addfa19f6 100644 --- a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp +++ b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h index c4a26ff79c5..a6052f5f6b6 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h +++ b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h @@ -3,7 +3,7 @@ #pragma once #include "iattributefilewriter.h" -#include +#include namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index cda2bea9fb1..cd2a423782e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include LOG_SETUP(".searchlib.attribute.enum_store_dictionary"); diff --git a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp index cf394623abd..e0ce0ef155e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp @@ -3,7 +3,7 @@ #include "enumattributesaver.h" #include "i_enum_store_dictionary.h" #include "iattributesavetarget.h" -#include +#include #include namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index e39ecc17aac..b76cf0c4192 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp index 51821389a92..fe19bf236aa 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp @@ -3,7 +3,7 @@ #include "multinumericattributesaver.h" #include "multivalueattributesaverutils.h" #include "multivalue.h" -#include +#include using vespalib::GenerationHandler; using search::multivalueattributesaver::CountWriter; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index eafa5bf0e1f..003fb9fa6b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp index 697dbcfd27b..16acf70eb59 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "multivalueattributesaverutils.h" -#include +#include namespace search::multivalueattributesaver { diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h index 213fd4c4777..e34f43deb0b 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h @@ -3,7 +3,7 @@ #pragma once #include "iattributesavetarget.h" -#include +#include #include namespace search::multivalueattributesaver { diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp index e0a6ea193e0..4385911d0b3 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp @@ -2,7 +2,7 @@ #include "reference_attribute_saver.h" #include -#include +#include #include "iattributesavetarget.h" diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp index afa893458b4..f3552db7c05 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp @@ -2,7 +2,7 @@ #include "singleenumattributesaver.h" #include -#include +#include #include "iattributesavetarget.h" diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp index 406cbbbe447..e362ecff6cd 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp index fd8d6162f01..362e1b45266 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp @@ -3,7 +3,7 @@ #include "dense_tensor_attribute_saver.h" #include "dense_tensor_store.h" #include "nearest_neighbor_index_saver.h" -#include +#include #include using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp index 2f0bb731aa4..81ec3a5218e 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp @@ -2,7 +2,7 @@ #include "generic_tensor_attribute_saver.h" #include "generic_tensor_store.h" -#include +#include #include using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp index acff30f8cbf..46a988d575e 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp @@ -2,7 +2,7 @@ #include "hnsw_index_saver.h" #include "hnsw_graph.h" -#include +#include namespace search::tensor { diff --git a/searchlib/src/vespa/searchlib/util/CMakeLists.txt b/searchlib/src/vespa/searchlib/util/CMakeLists.txt index ac5fc5f54fc..68dee99339f 100644 --- a/searchlib/src/vespa/searchlib/util/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/util/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_util OBJECT SOURCES + bufferwriter.cpp comprbuffer.cpp comprfile.cpp dirtraverse.cpp diff --git a/searchlib/src/vespa/searchlib/util/bufferwriter.cpp b/searchlib/src/vespa/searchlib/util/bufferwriter.cpp new file mode 100644 index 00000000000..6e57d6f58d4 --- /dev/null +++ b/searchlib/src/vespa/searchlib/util/bufferwriter.cpp @@ -0,0 +1,36 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "bufferwriter.h" + +namespace search { + +BufferWriter::BufferWriter() + : _cur(nullptr), + _end(nullptr), + _start(nullptr) +{ +} + +BufferWriter::~BufferWriter() = default; + +void +BufferWriter::writeSlow(const void *src, size_t len) +{ + size_t residue = len; + const char *csrc = static_cast(src); + for (;;) { + size_t maxLen = freeLen(); + if (residue <= maxLen) { + writeFast(csrc, residue); + break; + } + if (maxLen != 0) { + writeFast(csrc, maxLen); + csrc += maxLen; + residue -= maxLen; + } + flush(); + } +} + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/util/bufferwriter.h b/searchlib/src/vespa/searchlib/util/bufferwriter.h new file mode 100644 index 00000000000..3da6e3f8030 --- /dev/null +++ b/searchlib/src/vespa/searchlib/util/bufferwriter.h @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include + +namespace search { + +/** + * Abstract class to write to a buffer with an abstract backing store + * and abstract backing buffer. Each time backing buffer is full, + * flush() is called to resize it or drain it to the backing store. + */ +class BufferWriter +{ + char *_cur; + char *_end; + char *_start; +protected: + void rewind() { _cur = _start; } + + void setup(void *start, size_t len) { + _start = static_cast(start); + _end = _start + len; + rewind(); + } + + size_t freeLen() const { return _end - _cur; } + size_t usedLen() const { return _cur - _start; } + + void writeFast(const void *src, size_t len) + { + __builtin_memcpy(_cur, src, len); + _cur += len; + } + + void writeSlow(const void *src, size_t len); + +public: + BufferWriter(); + + virtual ~BufferWriter(); + + virtual void flush() = 0; + + void write(const void *src, size_t len) + { + if (__builtin_expect(len <= freeLen(), true)) { + writeFast(src, len); + return; + } + writeSlow(src, len); + } +}; + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h index 38d8337fab5..f0a58b0d25a 100644 --- a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h +++ b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h @@ -2,7 +2,7 @@ #pragma once -#include +#include #include #include diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index 33830bbe4ab..956d4c16d40 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -10,7 +10,6 @@ #include "unique_store_builder.hpp" #include "unique_store_dictionary.hpp" #include "unique_store_enumerator.hpp" -#include #include #include diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index e3397b54740..4029c4881c4 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -12,7 +12,6 @@ vespa_add_library(vespalib_vespalib_util OBJECT benchmark_timer.cpp blockingthreadstackexecutor.cpp box.cpp - bufferwriter.cpp classname.cpp closuretask.cpp compress.cpp diff --git a/vespalib/src/vespa/vespalib/util/bufferwriter.cpp b/vespalib/src/vespa/vespalib/util/bufferwriter.cpp deleted file mode 100644 index 6e57d6f58d4..00000000000 --- a/vespalib/src/vespa/vespalib/util/bufferwriter.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "bufferwriter.h" - -namespace search { - -BufferWriter::BufferWriter() - : _cur(nullptr), - _end(nullptr), - _start(nullptr) -{ -} - -BufferWriter::~BufferWriter() = default; - -void -BufferWriter::writeSlow(const void *src, size_t len) -{ - size_t residue = len; - const char *csrc = static_cast(src); - for (;;) { - size_t maxLen = freeLen(); - if (residue <= maxLen) { - writeFast(csrc, residue); - break; - } - if (maxLen != 0) { - writeFast(csrc, maxLen); - csrc += maxLen; - residue -= maxLen; - } - flush(); - } -} - -} // namespace search diff --git a/vespalib/src/vespa/vespalib/util/bufferwriter.h b/vespalib/src/vespa/vespalib/util/bufferwriter.h deleted file mode 100644 index 3da6e3f8030..00000000000 --- a/vespalib/src/vespa/vespalib/util/bufferwriter.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include - -namespace search { - -/** - * Abstract class to write to a buffer with an abstract backing store - * and abstract backing buffer. Each time backing buffer is full, - * flush() is called to resize it or drain it to the backing store. - */ -class BufferWriter -{ - char *_cur; - char *_end; - char *_start; -protected: - void rewind() { _cur = _start; } - - void setup(void *start, size_t len) { - _start = static_cast(start); - _end = _start + len; - rewind(); - } - - size_t freeLen() const { return _end - _cur; } - size_t usedLen() const { return _cur - _start; } - - void writeFast(const void *src, size_t len) - { - __builtin_memcpy(_cur, src, len); - _cur += len; - } - - void writeSlow(const void *src, size_t len); - -public: - BufferWriter(); - - virtual ~BufferWriter(); - - virtual void flush() = 0; - - void write(const void *src, size_t len) - { - if (__builtin_expect(len <= freeLen(), true)) { - writeFast(src, len); - return; - } - writeSlow(src, len); - } -}; - -} // namespace search -- cgit v1.2.3 From b2f1673a6d0f16368fe4528a7d1ff33eb6476506 Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Sun, 26 Apr 2020 09:51:38 +0200 Subject: Some additional tensor evaluation tests --- .../evaluation/EvaluationTestCase.java | 125 ++++++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 807eb3aa7ce..6a9108311b1 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -10,7 +10,6 @@ import com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.IfNode; -import com.yahoo.tensor.TensorType; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -389,6 +388,130 @@ public class EvaluationTestCase { } + @Test + public void testTile() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor(d0[2],d1[4]):[1,2,1,2,3,4,3,4]", + "tensor(d0[2],d1[4])(tensor0{input0:(d0 % 2), input1:(d1 % 2) } )", + "tensor(input0[2],input1[2]):[1, 2, 3, 4]", + "tensor(repeats0[2]):[1,2]"); + + tester.assertEvaluates("tensor(d0[6],d1[2]):[1,2,3,4,1,2,3,4,1,2,3,4]", + "tensor(d0[6],d1[2])(tensor0{input0:(d0 % 2), input1:(d1 % 2) } )", + "tensor(input0[2],input1[2]):[1, 2, 3, 4]", + "tensor(repeats0[2]):[3,1]"); + } + + @Test + public void testReshape() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor(d0[4]):[1,2,3,4]", + "tensor(d0[4])(tensor0{a0:(d0 / 2), a1:(d0 % 2)})", + "tensor(a0[2],a1[2]):[1,2,3,4]", + "tensor(d0[1]):[4]"); + + tester.assertEvaluates("tensor(d0[2],d1[2]):[1,2,3,4]", + "tensor(d0[2],d1[2])(tensor0{a0:(d0), a1:(d1)})", + "tensor(a0[2],a1[2]):[1,2,3,4]", + "tensor(d0[2]):[2,2]"); + + tester.assertEvaluates("tensor(d0[2],d1[1],d2[2]):[1,2,3,4]", + "tensor(d0[2],d1[1],d2[2])(tensor0{a0:(d0), a1:(d2)})", + "tensor(a0[2],a1[2]):[1,2,3,4]", + "tensor(d0[3]):[2,1,2]"); + + tester.assertEvaluates("tensor(d0[3],d1[2]):[1,2,3,4,5,6]", + "tensor(d0[3],d1[2])(tensor0{a0:0, a1:((d0 * 2 + d1) / 3), a2:((d0 * 2 + d1) % 3) })", + "tensor(a0[1],a1[2],a2[3]):[1,2,3,4,5,6]", + "tensor(d0[2]):[3,2]"); + + tester.assertEvaluates("tensor(d0[3],d1[2],d2[1],d3[1]):[1,2,3,4,5,6]", + "tensor(d0[3],d1[2],d2[1],d3[1])(tensor0{a0:0, a1:((d0 * 2 + d1) / 3), a2:((d0 * 2 + d1) % 3) })", + "tensor(a0[1],a1[2],a2[3]):[1,2,3,4,5,6]", + "tensor(d0[4]):[3,2,-1,1]"); + + } + + @Test + public void testMatmul() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor():{91}", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d0)", + "tensor(d0[6]):[1,2,3,4,5,6]", + "tensor(d0[6]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d1[2]):[22, 28]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d0)", + "tensor(d0[3]):[1,2,3]", + "tensor(d0[3],d1[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d1[2]):[22, 28]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d0)", + "tensor(d0[3],d1[2]):[1,2,3,4,5,6]", + "tensor(d0[3]):[1,2,3]"); + + tester.assertEvaluates("tensor(d0[2],d2[2]):[22,28,49,64]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d1)", + "tensor(d0[2],d1[3]):[1,2,3,4,5,6]", + "tensor(d1[3],d2[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[1],d1[2],d3[2]):[22,28,49,64]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d2)", + "tensor(d0[1],d1[2],d2[3]):[1,2,3,4,5,6]", + "tensor(d2[3],d3[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[1],d1[2],d3[2]):[22,28,49,64]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d2)", + "tensor(d1[2],d2[3]):[1,2,3,4,5,6]", + "tensor(d0[1],d2[3],d3[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[2],d1[2],d3[2]):[22,28,49,64,58,64,139,154]", + "reduce(join(tensor0{d0:0}, tensor1, f(x,y)(x*y)), sum, d2)", // notice peek + "tensor(d0[1],d1[2],d2[3]):[1,2,3,4,5,6]", + "tensor(d0[2],d2[3],d3[2]):[1,2,3,4,5,6,7,8,9,10,11,12]"); + + tester.assertEvaluates("tensor(d0[2],d1[2],d2[2],d4[2]):[22,28,49,64,58,64,139,154,76,100,103,136,220,244,301,334]", + "reduce(join(tensor0{d1:0}, tensor1{d0:0}, f(x,y)(x*y)), sum, d3)", // notice peeks + "tensor(d0[2],d1[1],d2[2],d3[3]):[1,2,3,4,5,6,7,8,9,10,11,12]", + "tensor(d0[1],d1[2],d3[3],d4[2]):[1,2,3,4,5,6,7,8,9,10,11,12]"); + + tester.assertEvaluates("tensor(d0[1],d1[4],d2[2],d4[2]):[22,28,49,64,220,244,301,334,634,676,769,820,1264,1324,1453,1522]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d3)", + "tensor(d0[1],d1[4],d2[2],d3[3]):[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24]", + "tensor(d0[1],d1[4],d3[3],d4[2]):[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24]"); + } + + @Test + public void testSplit() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor(d0[3]):[1,2,3]", + "tensor(d0[3])(tensor0{input0:(d0)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[3]):[4,5,6]", + "tensor(d0[3])(tensor0{input0:(d0+3)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[4]):[3,4,5,6]", + "tensor(d0[4])(tensor0{input0:(d0+2)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[2]):[3,4]", + "tensor(d0[2])(tensor0{input0:(d0+2)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[2]):[5,6]", + "tensor(d0[2])(tensor0{input0:(d0+4)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[1],d1[3]):[1,2,3]", + "tensor(d0[1],d1[3])(tensor0{input0:(d0), input1:(d1)} )", + "tensor(input0[2],input1[3]):[[1,2,3],[4,5,6]]"); + tester.assertEvaluates("tensor(d0[1],d1[3]):[4,5,6]", + "tensor(d0[1],d1[3])(tensor0{input0:(d0+1), input1:(d1)} )", + "tensor(input0[2],input1[3]):[[1,2,3],[4,5,6]]"); + } + @Test public void testTake() { EvaluationTester tester = new EvaluationTester(); -- cgit v1.2.3 From 440dd0a324f2b889ec8190a59d89545fbc9937f6 Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Sun, 26 Apr 2020 09:53:12 +0200 Subject: Limit size of serialization context for functions without arguments --- searchlib/abi-spec.json | 5 ++++- .../rankingexpression/ExpressionFunction.java | 5 ++++- .../rule/FunctionReferenceContext.java | 5 +++++ .../rankingexpression/rule/ReferenceNode.java | 23 ++++++++++++++++------ .../rule/SerializationContext.java | 6 ++++++ .../rankingexpression/rule/TensorFunctionNode.java | 10 ++++++++-- .../transform/ConstantDereferencer.java | 20 ++++++++++++++++--- 7 files changed, 61 insertions(+), 13 deletions(-) diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index d058104bf1b..6bce791914c 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1416,7 +1416,8 @@ "public com.yahoo.searchlib.rankingexpression.ExpressionFunction getFunction(java.lang.String)", "protected com.google.common.collect.ImmutableMap functions()", "public java.lang.String getBinding(java.lang.String)", - "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)" + "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)", + "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withoutBindings()" ], "fields": [] }, @@ -1573,7 +1574,9 @@ "public void addArgumentTypeSerialization(java.lang.String, java.lang.String, com.yahoo.tensor.TensorType)", "public void addFunctionTypeSerialization(java.lang.String, com.yahoo.tensor.TensorType)", "public com.yahoo.searchlib.rankingexpression.rule.SerializationContext withBindings(java.util.Map)", + "public com.yahoo.searchlib.rankingexpression.rule.SerializationContext withoutBindings()", "public java.util.Map serializedFunctions()", + "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withoutBindings()", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)" ], "fields": [] diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 674571ff73e..559f690a32d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -134,7 +134,10 @@ public class ExpressionFunction { for (int i = 0; i < arguments.size() && i < argumentValues.size(); ++i) { argumentBindings.put(arguments.get(i), argumentValues.get(i).toString(new StringBuilder(), context, path, null).toString()); } - return new Instance(toSymbol(argumentBindings), body.getRoot().toString(new StringBuilder(), context.withBindings(argumentBindings), path, null).toString()); + context = argumentBindings.isEmpty() ? context.withoutBindings() : context.withBindings(argumentBindings); + String symbol = toSymbol(argumentBindings); + String expressionString = body.getRoot().toString(new StringBuilder(), context, path, null).toString(); + return new Instance(symbol, expressionString); } /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java index 83aabada8f0..6717bec0258 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java @@ -68,4 +68,9 @@ public class FunctionReferenceContext { return new FunctionReferenceContext(this.functions, bindings); } + /** Returns a fresh context without bindings */ + public FunctionReferenceContext withoutBindings() { + return new FunctionReferenceContext(this.functions); + } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index 8fec3603f3e..335d3861d9d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -5,7 +5,6 @@ import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Context; -import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; @@ -13,7 +12,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.ArrayDeque; import java.util.Deque; import java.util.List; -import java.util.Optional; +import java.util.Map; /** * A node referring either to a value in the context or to a named ranking expression function. @@ -84,13 +83,25 @@ public final class ReferenceNode extends CompositeNode { if (path.contains(myPath)) throw new IllegalStateException("Cycle in ranking expression function: " + path); path.addLast(myPath); - ExpressionFunction.Instance instance = function.expand(context, getArguments().expressions(), path); + + String functionName = getName(); + String functionPropertyName = RankingExpression.propertyName(functionName); + boolean alreadySerialized = context.serializedFunctions().containsKey(functionPropertyName); + + if ( ! alreadySerialized || getArguments().size() > 0) { + ExpressionFunction.Instance instance = function.expand(context, getArguments().expressions(), path); + functionName = instance.getName(); + + context.addFunctionSerialization(RankingExpression.propertyName(functionName), instance.getExpressionString()); + for (Map.Entry argumentType : function.argumentTypes().entrySet()) + context.addArgumentTypeSerialization(functionName, argumentType.getKey(), argumentType.getValue()); + if (function.returnType().isPresent()) + context.addFunctionTypeSerialization(functionName, function.returnType().get()); + } path.removeLast(); - context.addFunctionSerialization(RankingExpression.propertyName(instance.getName()), instance.getExpressionString()); - return string.append("rankingExpression(").append(instance.getName()).append(')'); + return string.append("rankingExpression(").append(functionName).append(')'); } - // Not resolved in this context: output as-is return reference.toString(string, context, path, parent); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index d7807caa2b6..92889d6607b 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -95,6 +95,12 @@ public class SerializationContext extends FunctionReferenceContext { return new SerializationContext(functions(), bindings, this.serializedFunctions); } + /** Returns a fresh context without bindings */ + @Override + public SerializationContext withoutBindings() { + return new SerializationContext(functions(), null, this.serializedFunctions); + } + public Map serializedFunctions() { return serializedFunctions; } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java index 6e1cdf52158..befe2179dc1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java @@ -333,10 +333,16 @@ public class TensorFunctionNode extends CompositeNode { /** Returns a new context with the bindings replaced by the given bindings */ @Override public ExpressionToStringContext withBindings(Map bindings) { - return new ExpressionToStringContext(new SerializationContext(wrappedSerializationContext.functions().values(), bindings), - wrappedToStringContext, path, parent); + SerializationContext serializationContext = new SerializationContext(functions(), bindings, serializedFunctions()); + return new ExpressionToStringContext(serializationContext, wrappedToStringContext, path, parent); } + /** Returns a fresh context without bindings */ + @Override + public SerializationContext withoutBindings() { + SerializationContext serializationContext = new SerializationContext(functions(), null, serializedFunctions()); + return new ExpressionToStringContext(serializationContext, null, path, parent); + } } /** Turns an EvaluationContext into a Context */ diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java index a541eac2421..fed4f60e983 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.transform; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; @@ -28,8 +28,15 @@ public class ConstantDereferencer extends ExpressionTransformer 0) { return node; // not a number constant reference } -- cgit v1.2.3 From 4b7307e121c6f292c5f15e982f19eff303d2d005 Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Sun, 26 Apr 2020 09:55:48 +0200 Subject: Avoid re-resolving types of diamond-shaped dependencies --- .../searchdefinition/MapEvaluationTypeContext.java | 34 +++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index 6de7c985326..4011ce43841 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -41,6 +41,9 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement private final Map resolvedTypes = new HashMap<>(); + /** To avoid re-resolving diamond-shaped dependencies */ + private final Map globallyResolvedTypes; + /** For invocation loop detection */ private final Deque currentResolutionCallStack; @@ -53,6 +56,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement this.currentResolutionCallStack = new ArrayDeque<>(); this.queryFeaturesNotDeclared = new TreeSet<>(); tensorsAreUsed = false; + globallyResolvedTypes = new HashMap<>(); } private MapEvaluationTypeContext(Map functions, @@ -60,12 +64,14 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement Map featureTypes, Deque currentResolutionCallStack, SortedSet queryFeaturesNotDeclared, - boolean tensorsAreUsed) { + boolean tensorsAreUsed, + Map globallyResolvedTypes) { super(functions, bindings); this.featureTypes.putAll(featureTypes); this.currentResolutionCallStack = currentResolutionCallStack; this.queryFeaturesNotDeclared = queryFeaturesNotDeclared; this.tensorsAreUsed = tensorsAreUsed; + this.globallyResolvedTypes = globallyResolvedTypes; } public void setType(Reference reference, TensorType type) { @@ -82,11 +88,25 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement resolvedTypes.clear(); } - @Override + private boolean referenceCanBeResolvedGlobally(Reference reference) { + Optional function = functionInvocation(reference); + return function.isPresent() && function.get().arguments().size() == 0; + // are there other cases we would like to resolve globally? + } + + @Override public TensorType getType(Reference reference) { // computeIfAbsent without concurrent modification due to resolve adding more resolved entries: + + boolean canBeResolvedGlobally = referenceCanBeResolvedGlobally(reference); + TensorType resolvedType = resolvedTypes.get(reference); - if (resolvedType != null) return resolvedType; + if (resolvedType == null && canBeResolvedGlobally) { + resolvedType = globallyResolvedTypes.get(reference); + } + if (resolvedType != null) { + return resolvedType; + } resolvedType = resolveType(reference); if (resolvedType == null) @@ -94,6 +114,11 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement resolvedTypes.put(reference, resolvedType); if (resolvedType.rank() > 0) tensorsAreUsed = true; + + if (canBeResolvedGlobally) { + globallyResolvedTypes.put(reference, resolvedType); + } + return resolvedType; } @@ -254,7 +279,8 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement featureTypes, currentResolutionCallStack, queryFeaturesNotDeclared, - tensorsAreUsed); + tensorsAreUsed, + globallyResolvedTypes); } } -- cgit v1.2.3 From ee76bcb34de37f748d4377c0dc2d4899a0c41d6a Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Sun, 26 Apr 2020 09:58:26 +0200 Subject: Add functions used by firstphase, secondphase and summary first --- .../com/yahoo/searchdefinition/RankProfile.java | 5 ++-- .../searchdefinition/derived/RawRankProfile.java | 35 ++++++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 23eb814de81..ea126123a25 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -680,11 +680,12 @@ public class RankProfile implements Cloneable { Map inlineFunctions = compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + // Function compiling second pass: compile all functions and insert previously compiled inline functions functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); } private void checkNameCollisions(Map functions, Map constants) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 1a22b98fd9f..da2b23595a9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -17,6 +17,7 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; +import org.tensorflow.op.core.Rank; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -200,9 +201,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (functions.isEmpty()) return; List functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); - Map functionProperties = new LinkedHashMap<>(); - functionProperties.putAll(deriveFunctionProperties(functions, functionExpressions)); if (firstPhaseRanking != null) { functionProperties.putAll(firstPhaseRanking.getRankProperties(functionExpressions)); @@ -211,20 +210,30 @@ public class RawRankProfile implements RankProfilesConfig.Producer { functionProperties.putAll(secondPhaseRanking.getRankProperties(functionExpressions)); } + SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); + replaceFunctionSummaryFeatures(context); + + // First phase, second phase and summary features should add all required functions to the context. + // However, we need to add any functions not referenced in those anyway for model-evaluation. + deriveFunctionProperties(functions, functionExpressions, functionProperties); + for (Map.Entry e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); } - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); - replaceFunctionSummaryFeatures(context); } - private Map deriveFunctionProperties(Map functions, - List functionExpressions) { - SerializationContext context = new SerializationContext(functionExpressions); + private void deriveFunctionProperties(Map functions, + List functionExpressions, + Map functionProperties) { + SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); for (Map.Entry e : functions.entrySet()) { + String propertyName = RankingExpression.propertyName(e.getKey()); + if (context.serializedFunctions().containsKey(propertyName)) { + continue; + } String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); - context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expressionString); + context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expressionString); for (Map.Entry argumentType : e.getValue().function().argumentTypes().entrySet()) context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue()); if (e.getValue().function().returnType().isPresent()) @@ -232,7 +241,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { // else if (e.getValue().function().arguments().isEmpty()) TODO: Enable this check when we resolve all types // throw new IllegalStateException("Type of function '" + e.getKey() + "' is not resolved"); } - return context.serializedFunctions(); + functionProperties.putAll(context.serializedFunctions()); } private void replaceFunctionSummaryFeatures(SerializationContext context) { @@ -241,9 +250,11 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (Iterator i = summaryFeatures.iterator(); i.hasNext(); ) { ReferenceNode referenceNode = i.next(); // Is the feature a function? - if (context.getFunction(referenceNode.getName()) != null) { - context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()), - referenceNode.toString(new StringBuilder(), context, null, null).toString()); + ExpressionFunction function = context.getFunction(referenceNode.getName()); + if (function != null) { + String propertyName = RankingExpression.propertyName(referenceNode.getName()); + String expressionString = function.getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + context.addFunctionSerialization(propertyName, expressionString); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode); i.remove(); // Will add the expanded one in next block -- cgit v1.2.3 From 12b09c46a3e32df3a90d326f30171bc789d1bcb8 Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Sun, 26 Apr 2020 10:01:15 +0200 Subject: Fix unit tests --- .../RankingExpressionShadowingTestCase.java | 47 +++++++++++----------- .../processing/RankProfileSearchFixture.java | 8 ++++ .../RankingExpressionWithTensorFlowTestCase.java | 4 +- .../processing/RankingExpressionsTestCase.java | 16 ++++---- .../com/yahoo/vespa/model/ml/MlModelsTest.java | 4 +- 5 files changed, 43 insertions(+), 36 deletions(-) diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 5c1134f928c..e4ca83640e9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -50,10 +50,10 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(sin).rankingScript,x * x)", - testRankProperties.get(0).toString()); assertEquals("(rankingExpression(sin@).rankingScript,2 * 2)", - censorBindingHash(testRankProperties.get(1).toString())); + censorBindingHash(testRankProperties.get(0).toString())); + assertEquals("(rankingExpression(sin).rankingScript,x * x)", + testRankProperties.get(1).toString()); assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", censorBindingHash(testRankProperties.get(2).toString())); } @@ -94,27 +94,26 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(tan@).rankingScript,2 * 2)", + censorBindingHash(testRankProperties.get(0).toString())); + assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(sin@).rankingScript,rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(2).toString())); assertEquals("(rankingExpression(tan).rankingScript,x * x)", - testRankProperties.get(0).toString()); + testRankProperties.get(3).toString()); assertEquals("(rankingExpression(tan@).rankingScript,x * x)", - censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(cos).rankingScript,rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(3).toString())); - assertEquals("(rankingExpression(sin).rankingScript,rankingExpression(cos@))", censorBindingHash(testRankProperties.get(4).toString())); - assertEquals("(rankingExpression(tan@).rankingScript,2 * 2)", + assertEquals("(rankingExpression(cos).rankingScript,rankingExpression(tan@))", censorBindingHash(testRankProperties.get(5).toString())); assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(6).toString())); - assertEquals("(rankingExpression(sin@).rankingScript,rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(6).toString())); + assertEquals("(rankingExpression(sin).rankingScript,rankingExpression(cos@))", censorBindingHash(testRankProperties.get(7).toString())); assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", censorBindingHash(testRankProperties.get(8).toString())); } - @Test public void testFunctionShadowingArguments() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); @@ -144,12 +143,12 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(sin).rankingScript,x * x)", - testRankProperties.get(0).toString()); assertEquals("(rankingExpression(sin@).rankingScript,4.0 * 4.0)", - censorBindingHash(testRankProperties.get(1).toString())); + censorBindingHash(testRankProperties.get(0).toString())); assertEquals("(rankingExpression(sin@).rankingScript,cos(5.0) * cos(5.0))", - censorBindingHash(testRankProperties.get(2).toString())); + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(sin).rankingScript,x * x)", + testRankProperties.get(2).toString()); assertEquals("(vespa.rank.firstphase,rankingExpression(firstphase))", censorBindingHash(testRankProperties.get(3).toString())); assertEquals("(rankingExpression(firstphase).rankingScript,cos(rankingExpression(sin@)) + rankingExpression(sin@))", @@ -208,17 +207,17 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { queryProfiles, new ImportedMlModels(), new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", - testRankProperties.get(0).toString()); assertEquals("(rankingExpression(relu@).rankingScript,max(1.0,reduce(query(q) * constant(W_hidden), sum, input) + constant(b_input)))", - censorBindingHash(testRankProperties.get(1).toString())); + censorBindingHash(testRankProperties.get(0).toString())); assertEquals("(rankingExpression(hidden_layer).rankingScript,rankingExpression(relu@))", - censorBindingHash(testRankProperties.get(2).toString())); + censorBindingHash(testRankProperties.get(1).toString())); assertEquals("(rankingExpression(hidden_layer).type,tensor(x[]))", - censorBindingHash(testRankProperties.get(3).toString())); + censorBindingHash(testRankProperties.get(2).toString())); assertEquals("(rankingExpression(final_layer).rankingScript,sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))", - testRankProperties.get(4).toString()); + testRankProperties.get(3).toString()); assertEquals("(rankingExpression(final_layer).type,tensor(x[]))", + testRankProperties.get(4).toString()); + assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", testRankProperties.get(5).toString()); assertEquals("(vespa.rank.secondphase,rankingExpression(secondphase))", testRankProperties.get(6).toString()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 0cd6674751e..e6616ce0dd1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -41,6 +41,14 @@ class RankProfileSearchFixture { private Search search; private Map compiledRankProfiles = new HashMap<>(); + public RankProfileRegistry getRankProfileRegistry() { + return rankProfileRegistry; + } + + public QueryProfileRegistry getQueryProfileRegistry() { + return queryProfileRegistry; + } + RankProfileSearchFixture(String rankProfiles) throws ParseException { this(MockApplicationPackage.createEmpty(), new QueryProfileRegistry(), rankProfiles); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index a64a964727c..680f2dd9659 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -319,7 +319,7 @@ public class RankingExpressionWithTensorFlowTestCase { @Test public void testFunctionGeneration() { final String name = "mnist_saved"; - final String expression = "join(join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String expression = "join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b))"; final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; final String functionExpression2 = "join(reduce(join(join(join(0.009999999776482582, imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; @@ -349,7 +349,7 @@ public class RankingExpressionWithTensorFlowTestCase { " rank-profile my_profile_child inherits my_profile {\n" + " }"; - final String expression = "join(join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String expression = "join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b))"; final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; final String functionExpression2 = "join(reduce(join(join(join(0.009999999776482582, imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index b3eda9b7e13..1567a4c3b5e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -44,20 +44,20 @@ public class RankingExpressionsTestCase extends SchemaTestCase { new AttributeFields(search)).configProperties(); assertEquals(6, rankProperties.size()); - assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(0).getFirst()); - assertEquals("var1 * var2 + 890", rankProperties.get(0).getSecond()); + assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(2).getFirst()); + assertEquals("var1 * var2 + 890", rankProperties.get(2).getSecond()); - assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(1).getFirst()); - assertEquals("78 + closeness(distance)", rankProperties.get(1).getSecond()); + assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(3).getFirst()); + assertEquals("78 + closeness(distance)", rankProperties.get(3).getSecond()); assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst()); assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond()); - assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(3).getFirst()); - assertEquals("7 * 8 + 890", rankProperties.get(3).getSecond()); + assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst()); + assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond()); - assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(2).getFirst()); - assertEquals("4 * 5 + 890", rankProperties.get(2).getSecond()); + assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); + assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); } @Test(expected = IllegalArgumentException.class) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java index ca84eb5eed7..57dbf132883 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java @@ -64,10 +64,10 @@ public class MlModelsTest { private final String testProfile = "rankingExpression(input).rankingScript: attribute(argument)\n" + "rankingExpression(input).type: tensor(d0[],d1[784])\n" + - "rankingExpression(Placeholder).rankingScript: attribute(argument)\n" + - "rankingExpression(Placeholder).type: tensor(d0[],d1[784])\n" + "rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add).rankingScript: join(reduce(join(rename(rankingExpression(input), (d0, d1), (d0, d4)), constant(mnist_saved_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(mnist_saved_dnn_hidden1_bias_read), f(a,b)(a + b))\n" + "rankingExpression(mnist_tensorflow).rankingScript: join(reduce(join(map(join(reduce(join(join(join(0.009999999776482582, rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add), f(a,b)(a * b)), rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add), f(a,b)(max(a,b))), constant(mnist_saved_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(mnist_saved_dnn_hidden2_bias_read), f(a,b)(a + b)), f(a)(1.0507009873554805 * if (a >= 0, a, 1.6732632423543772 * (exp(a) - 1)))), constant(mnist_saved_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(mnist_saved_dnn_outputs_bias_read), f(a,b)(a + b))\n" + + "rankingExpression(Placeholder).rankingScript: attribute(argument)\n" + + "rankingExpression(Placeholder).type: tensor(d0[],d1[784])\n" + "rankingExpression(mnist_softmax_tensorflow).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))\n" + "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" + "rankingExpression(my_xgboost).rankingScript: if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)\n" + -- cgit v1.2.3 From 180cb44c805d3c2703041c0e34d2e4bfb8b67915 Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Sun, 26 Apr 2020 10:02:20 +0200 Subject: Ensure dimensions to matmul over are innermost --- .../java/ai/vespa/rankingexpression/importer/operations/MatMul.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java index 2b0af93fd8e..d326291a5a5 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java @@ -126,7 +126,7 @@ public class MatMul extends IntermediateOperation { // a1 < a2 < a3 < a4 for (int j = i+1; j < typeA.rank(); ++j) { String jDim = typeA.dimensionNames().get(j); - renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(true), this); + renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(false), this); } // not equal to last 2 dimensions in B for (int j = typeB.rank()-2; j < typeB.rank(); ++j) { @@ -148,7 +148,7 @@ public class MatMul extends IntermediateOperation { // b1 < b2 < b3 < b4 for (int j = i+1; j < typeB.rank(); ++j) { String jDim = typeB.dimensionNames().get(j); - renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(true), this); + renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(false), this); } // not equal to last 2 dimensions in A for (int j = typeA.rank()-2; j < typeA.rank(); ++j) { -- cgit v1.2.3 From d95c4ceb6c4db635e93b605d719f01db9d5f2e6f Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Sun, 26 Apr 2020 10:06:28 +0200 Subject: Remove unused import --- .../src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java | 1 - 1 file changed, 1 deletion(-) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index da2b23595a9..c3c10139684 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -17,7 +17,6 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; -import org.tensorflow.op.core.Rank; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -- cgit v1.2.3 From a7ca18c57e6fdfce0d06db7efb41f87a2049206e Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Sun, 26 Apr 2020 12:25:20 +0200 Subject: Smarter window change handling --- .../java/com/yahoo/config/provision/Capacity.java | 7 ++- .../com/yahoo/config/provision/ClusterSpec.java | 2 +- .../provisioning/NodeRepositoryProvisioner.java | 54 ++++++++++++++++----- .../provisioning/MultigroupProvisioningTest.java | 2 +- .../provision/provisioning/ProvisioningTest.java | 56 ++++++++++++++++++++++ .../provision/provisioning/ProvisioningTester.java | 25 ++++++++++ 6 files changed, 131 insertions(+), 15 deletions(-) diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 3c7a14b9496..f723575c342 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -72,7 +72,12 @@ public final class Capacity { /** Create a non-required, failable capacity request */ public static Capacity from(ClusterResources resources) { - return from(resources, false, true); + return from(resources, resources); + } + + /** Create a non-required, failable capacity request */ + public static Capacity from(ClusterResources min, ClusterResources max) { + return from(min, max, false, true); } public static Capacity from(ClusterResources resources, boolean required, boolean canFail) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 97549e851ad..3a230c89732 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -29,7 +29,7 @@ public final class ClusterSpec { this.type = type; this.id = id; this.groupId = groupId; - this.vespaVersion = vespaVersion; + this.vespaVersion = Objects.requireNonNull(vespaVersion); this.exclusive = exclusive; // TODO(mpolden): Require combinedId to always be present for type combined after April 2020 if (type != Type.combined && combinedId.isPresent()) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index e3b7c62cc0d..9ffc4d0efac 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -137,32 +137,62 @@ public class NodeRepositoryProvisioner implements Provisioner { application = application.withClusterLimits(clusterId, requested.minResources(), requested.maxResources()); nodeRepository.applications().put(application, lock); return application.clusters().get(clusterId).targetResources() - .orElseGet(() -> currentResources(applicationId, clusterId, requested) - .orElse(requested.minResources())); + .orElseGet(() -> currentResources(applicationId, clusterId, requested)); } } - /** Returns the current resources of this cluster, if it's already deployed and inside the requested limits */ - private Optional currentResources(ApplicationId applicationId, - ClusterSpec.Id clusterId, - Capacity requested) { + /** Returns the current resources of this cluster, or the closes */ + private ClusterResources currentResources(ApplicationId applicationId, + ClusterSpec.Id clusterId, + Capacity requested) { List nodes = NodeList.copyOf(nodeRepository.getNodes(applicationId, Node.State.active)) .cluster(clusterId) .not().retired() .not().removable() .asList(); - if (nodes.isEmpty()) return Optional.empty(); - long groups = nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); + if (nodes.isEmpty()) return requested.minResources(); // New deployment: Start at min - // To allow non-numeric settings to be updated without resetting to the min target, we need to use - // the non-numeric settings of the current min limit with the current numeric settings + long groups = nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); + // Settings which are not autoscaled should always be taken from the currently requested capacity + // and for those min and max are always the same NodeResources nodeResources = nodes.get(0).allocation().get().requestedResources() + .withBandwidthGbps(requested.minResources().nodeResources().bandwidthGbps()) .with(requested.minResources().nodeResources().diskSpeed()) .with(requested.minResources().nodeResources().storageType()); var currentResources = new ClusterResources(nodes.size(), (int)groups, nodeResources); - if ( ! currentResources.isWithin(requested.minResources(), requested.maxResources())) return Optional.empty(); + return ensureWithin(requested.minResources(), requested.maxResources(), currentResources); + } + + /** Make the minimal adjustments needed to the current resources to stay within the limits */ + private ClusterResources ensureWithin(ClusterResources min, ClusterResources max, ClusterResources current) { + int nodes = between(min.nodes(), max.nodes(), current.nodes()); + int groups = between(min.groups(), max.groups(), current.groups()); + if (nodes % groups != 0) { + // That didn't work - try to preserve current group size instead. + // Rounding here is needed because a node may be missing due to node failing. + int currentGroupsSize = Math.round((float)current.nodes() / current.groups()); + nodes = currentGroupsSize * groups; + if (nodes != between(min.nodes(), max.nodes(), nodes)) { + // Give up: Use max + nodes = max.nodes(); + groups = max.groups(); + } + } + double vcpu = between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), current.nodeResources().vcpu()); + double memoryGb = between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), current.nodeResources().memoryGb()); + double diskGb = between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), current.nodeResources().diskGb()); + NodeResources nodeResources = current.nodeResources().withVcpu(vcpu) + .withMemoryGb(memoryGb) + .withDiskGb(diskGb); + return new ClusterResources(nodes, groups, nodeResources); + } + + private int between(int min, int max, int n) { + return Math.min(max, Math.max(min, n)); + } - return Optional.of(currentResources); + private double between(double min, double max, double n) { + return Math.min(max, Math.max(min, n)); } private void logIfDownscaled(int targetNodes, int actualNodes, ClusterSpec cluster, ProvisionLogger logger) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index 1265961e351..050f3b7e865 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -109,7 +109,7 @@ public class MultigroupProvisioningTest { } @Test - public void test_one_node_and_group_to_two_with_flavor_migration() { + public void test_one_node_and_group_to_two_with_resource_change() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).build(); ApplicationId application1 = tester.makeApplicationId(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 8fc73d68785..1b7c75a03ac 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -405,6 +405,58 @@ public class ProvisioningTest { tester.activate(application, state.allHosts); } + @Test + public void test_changing_limits() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + tester.makeReadyHosts(30, new NodeResources(20, 40, 100, 4)); + tester.deployZoneApp(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + // Initial deployment + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 20), + resources(8, 4, 4, 20, 40))); + tester.assertNodes("Initial allocation at min", + 4, 2, 2, 10, 20, + app1, cluster1); + + // Move window above current allocation + tester.activate(app1, cluster1, Capacity.from(resources(8, 4, 4, 20, 40), + resources(10, 5, 5, 25, 50))); + tester.assertNodes("New allocation at new min", + 8, 4, 4, 20, 40, + app1, cluster1); + + // Move window below current allocation + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 20), + resources(6, 3, 3, 15, 25))); + tester.assertNodes("New allocation at new max", + 6, 3, 3, 15, 25, + app1, cluster1); + + // Widening window does not change allocation + tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 1, 5, 15), + resources(8, 4, 4, 20, 30))); + tester.assertNodes("Same allocation", + 6, 3, 3, 15, 25, + app1, cluster1); + + // Changing limits in opposite directions cause a mixture of min and max + tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 10, 30, 5), + resources(4, 2, 14, 40, 10))); + tester.assertNodes("A mix of min and max", + 4, 2, 10, 30, 10, + app1, cluster1); + + // Changing group size + tester.activate(app1, cluster1, Capacity.from(resources(6, 3, 8, 25, 5), + resources(9, 3, 12, 35, 15))); + tester.assertNodes("Groups changed", + 6, 3, 10, 30, 10, + app1, cluster1); + } + @Test(expected = IllegalArgumentException.class) public void prod_deployment_requires_redundancy() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); @@ -920,4 +972,8 @@ public class ProvisioningTest { .isPresent(); } + private ClusterResources resources(int nodes, int groups, double vcpu, double memory, double disk) { + return new ClusterResources(nodes, groups, new NodeResources(vcpu, memory, disk, 0.1)); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index a8df47aab1a..cb4509b3de7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -149,6 +149,10 @@ public class ProvisioningTester { return hosts1; } + public Collection activate(ApplicationId application, ClusterSpec cluster, Capacity capacity) { + return activate(application, prepare(application, cluster, capacity, true)); + } + public Collection activate(ApplicationId application, Collection hosts) { NestedTransaction transaction = new NestedTransaction(); transaction.add(new CuratorTransaction(curator)); @@ -197,6 +201,22 @@ public class ProvisioningTester { } } + /** Assert on the current *non retired* nodes */ + public void assertNodes(String explanation, int nodes, int groups, double vcpu, double memory, double disk, + ApplicationId app, ClusterSpec cluster) { + List nodeList = nodeRepository.list().owner(app).cluster(cluster.id()).not().retired().asList(); + assertEquals(explanation + ": Node count", + nodes, + nodeList.size()); + assertEquals(explanation + ": Group count", + groups, + nodeList.stream().map(n -> n.allocation().get().membership().cluster().group().get()).distinct().count()); + for (Node node : nodeList) + assertEquals(explanation + ": Resources", + new NodeResources(vcpu, memory, disk, 0.1), + node.flavor().resources()); + } + public void fail(HostSpec host) { int beforeFailCount = nodeRepository.getNode(host.hostname(), Node.State.active).get().status().failCount(); Node failedNode = nodeRepository.fail(host.hostname(), Agent.system, "Failing to unit test"); @@ -241,6 +261,11 @@ public class ProvisioningTester { return makeReadyNodes(n, flavor, NodeType.tenant); } + /** Call depliyZoneApp() after this before deploying applications */ + public List makeReadyHosts(int n, NodeResources resources) { + return makeReadyNodes(n, resources, NodeType.host, 5); + } + public List makeReadyNodes(int n, NodeResources resources) { return makeReadyNodes(n, resources, NodeType.tenant, 0); } -- cgit v1.2.3 From ae319dc3fd55f0aa7a804c51a62a194ab3b3c822 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Sun, 26 Apr 2020 17:10:46 +0200 Subject: Allocate on hosts in mosts tests --- .../maintenance/LoadBalancerExpirerTest.java | 2 +- .../provision/provisioning/ProvisioningTest.java | 64 +++++++++++----------- .../provision/provisioning/ProvisioningTester.java | 7 ++- 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index e278cdc992a..a5e96369591 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -155,7 +155,7 @@ public class LoadBalancerExpirerTest { } private void deployApplication(ApplicationId application, boolean activate, ClusterSpec.Id... clusters) { - tester.makeReadyNodes(10, "d-1-4-10"); + tester.makeReadyNodes(10, new NodeResources(1, 4, 10, 0.3)); List hosts = new ArrayList<>(); for (var cluster : clusters) { hosts.addAll(tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster).vespaVersion(Vtag.currentVersion).build(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 1b7c75a03ac..463a3ef3fb9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -59,7 +59,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); ApplicationId application2 = tester.makeApplicationId(); - tester.makeReadyNodes(21, defaultResources); + tester.makeReadyHosts(21, defaultResources).deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); @@ -137,7 +137,7 @@ public class ProvisioningTest { public void nodeVersionIsReturnedIfSet() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); // deploy @@ -162,7 +162,7 @@ public class ProvisioningTest { public void dockerImageRepoIsReturnedIfSet() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); // deploy @@ -192,7 +192,8 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(24, defaultResources); + tester.makeReadyHosts(24, defaultResources); + tester.deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); @@ -262,18 +263,18 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(12, small); + tester.makeReadyHosts(12, small); + tester.makeReadyHosts(16, large); + tester.deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 2, 4, 4, small, tester); tester.activate(application1, state1.allHosts); - // redeploy with reduced size (to cause us to have retired nodes before switching flavor) + // redeploy with reduced size (to cause us to have retired nodes before switching resources) SystemState state2 = prepare(application1, 2, 2, 3, 3, small, tester); tester.activate(application1, state2.allHosts); - tester.makeReadyNodes(16, large); - // redeploy with increased sizes and new flavor SystemState state3 = prepare(application1, 3, 4, 4, 5, large, tester); assertEquals("New nodes are reserved", 16, tester.nodeRepository().getNodes(application1, Node.State.reserved).size()); @@ -292,7 +293,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(5, defaultResources); + tester.makeReadyHosts(5, defaultResources).deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 0, 3, 0, defaultResources, tester); @@ -315,7 +316,7 @@ public class ProvisioningTest { public void dev_deployment_node_size() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -327,8 +328,8 @@ public class ProvisioningTest { @Test public void requested_resources_info_is_retained() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + tester.makeReadyHosts(13, defaultResources).deployZoneApp(); - tester.makeReadyNodes(13, defaultResources, NodeType.host, 1); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -371,7 +372,7 @@ public class ProvisioningTest { public void deploy_specific_vespa_version() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -384,7 +385,7 @@ public class ProvisioningTest { public void deploy_specific_vespa_version_and_docker_image() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -399,7 +400,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(4, defaultResources); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -408,8 +409,7 @@ public class ProvisioningTest { @Test public void test_changing_limits() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyHosts(30, new NodeResources(20, 40, 100, 4)); - tester.deployZoneApp(); + tester.makeReadyHosts(30, new NodeResources(20, 40, 100, 4)).deployZoneApp(); ApplicationId app1 = tester.makeApplicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); @@ -462,7 +462,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); prepare(application, 1, 2, 3, 3, defaultResources, tester); } @@ -471,7 +471,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); try { prepare(application, 2, 2, 3, 3, new NodeResources(2, 2, 10, 2), tester); @@ -486,7 +486,7 @@ public class ProvisioningTest { public void dev_deployment_flavor() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, new NodeResources(2, 4, 10, 2), NodeType.host, 1); + tester.makeReadyHosts(4, new NodeResources(2, 4, 10, 2)); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -504,7 +504,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(4, large); + tester.makeReadyHosts(4, large).deployZoneApp(); SystemState state = prepare(application, 2, 2, 3, 3, large, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -515,7 +515,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.staging, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(14, defaultResources); + tester.makeReadyHosts(14, defaultResources).deployZoneApp(); SystemState state = prepare(application, 1, 1, 1, 64, defaultResources, tester); // becomes 1, 1, 1, 1, 6 assertEquals(9, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -525,7 +525,7 @@ public class ProvisioningTest { public void activate_after_reservation_timeout() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); @@ -547,7 +547,7 @@ public class ProvisioningTest { public void out_of_capacity() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(9, defaultResources); // need 2+2+3+3=10 + tester.makeReadyHosts(9, defaultResources).deployZoneApp(); // need 2+2+3+3=10 ApplicationId application = tester.makeApplicationId(); try { prepare(application, 2, 2, 3, 3, defaultResources, tester); @@ -564,7 +564,7 @@ public class ProvisioningTest { Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(13, defaultResources); + tester.makeReadyHosts(13, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); try { prepare(application, 2, 2, 6, 3, defaultResources, tester); @@ -582,7 +582,7 @@ public class ProvisioningTest { Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(13, defaultResources); + tester.makeReadyHosts(13, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); prepare(application, 2, 2, 6, 3, defaultResources, tester); } @@ -590,7 +590,7 @@ public class ProvisioningTest { @Test public void out_of_capacity_but_cannot_fail() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified), false, false)); @@ -624,7 +624,7 @@ public class ProvisioningTest { ApplicationId application = tester.makeApplicationId(); // Create 10 nodes - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); // Allocate 5 nodes ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.activate(application, tester.prepare(application, cluster, capacity)); @@ -655,7 +655,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(14, defaultResources); + tester.makeReadyHosts(14, defaultResources).deployZoneApp(); // deploy SystemState state1 = prepare(application1, 3, 3, 4, 4, defaultResources, tester); @@ -683,7 +683,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); // Deploy application { @@ -711,7 +711,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(2, defaultResources); + tester.makeReadyHosts(2, defaultResources).deployZoneApp(); // Deploy fails with out of capacity try { @@ -722,7 +722,7 @@ public class ProvisioningTest { tester.getNodes(application, Node.State.reserved).size()); // Enough nodes become available - tester.makeReadyNodes(2, defaultResources); + tester.makeReadyHosts(2, defaultResources).deployZoneApp(); // Deploy is retried after a few minutes tester.clock().advance(Duration.ofMinutes(2)); @@ -792,7 +792,7 @@ public class ProvisioningTest { var tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); var application = tester.makeApplicationId(); - tester.makeReadyNodes(4, defaultResources); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); // Application allocates two content nodes initially, with cluster type content ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index cb4509b3de7..cbb15311867 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -261,9 +261,10 @@ public class ProvisioningTester { return makeReadyNodes(n, flavor, NodeType.tenant); } - /** Call depliyZoneApp() after this before deploying applications */ - public List makeReadyHosts(int n, NodeResources resources) { - return makeReadyNodes(n, resources, NodeType.host, 5); + /** Call deployZoneApp() after this before deploying applications */ + public ProvisioningTester makeReadyHosts(int n, NodeResources resources) { + makeReadyNodes(n, resources, NodeType.host, 5); + return this; } public List makeReadyNodes(int n, NodeResources resources) { -- cgit v1.2.3 From 30ecdbb7ea88f17d24875e18ac45fe662610de77 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Sun, 26 Apr 2020 18:48:54 +0200 Subject: Cleanup naming and locations --- .../noderepository/RealNodeRepositoryTest.java | 3 +- .../src/main/config/node-repository.xml | 4 +- .../vespa/hosted/provision/NodeRepository.java | 2 +- .../provision/restapi/ApplicationSerializer.java | 47 + .../provision/restapi/HostCapacityResponse.java | 164 +++ .../hosted/provision/restapi/JobsResponse.java | 42 + .../provision/restapi/LoadBalancersResponse.java | 89 ++ .../restapi/LoadBalancersV1ApiHandler.java | 53 + .../hosted/provision/restapi/NodeAclResponse.java | 96 ++ .../hosted/provision/restapi/NodePatcher.java | 270 +++++ .../provision/restapi/NodeResourcesSerializer.java | 57 ++ .../hosted/provision/restapi/NodeSerializer.java | 76 ++ .../hosted/provision/restapi/NodesResponse.java | 233 +++++ .../provision/restapi/NodesV2ApiHandler.java | 436 ++++++++ .../provision/restapi/NotFoundException.java | 15 + .../hosted/provision/restapi/UpgradeResponse.java | 54 + .../restapi/v2/ApplicationSerializer.java | 48 - .../provision/restapi/v2/HostCapacityResponse.java | 164 --- .../hosted/provision/restapi/v2/JobsResponse.java | 42 - .../restapi/v2/LoadBalancersApiHandler.java | 53 - .../restapi/v2/LoadBalancersResponse.java | 89 -- .../provision/restapi/v2/NodeAclResponse.java | 96 -- .../hosted/provision/restapi/v2/NodePatcher.java | 270 ----- .../restapi/v2/NodeResourcesSerializer.java | 57 -- .../provision/restapi/v2/NodeSerializer.java | 76 -- .../provision/restapi/v2/NodesApiHandler.java | 436 -------- .../hosted/provision/restapi/v2/NodesResponse.java | 233 ----- .../provision/restapi/v2/NotFoundException.java | 15 - .../provision/restapi/v2/UpgradeResponse.java | 54 - .../provision/testutils/ContainerConfig.java | 4 +- .../provision/restapi/LoadBalancersV1ApiTest.java | 30 + .../provision/restapi/NodeSerializerTest.java | 31 + .../hosted/provision/restapi/NodesV2ApiTest.java | 984 ++++++++++++++++++ .../hosted/provision/restapi/RestApiTester.java | 96 ++ .../restapi/responses/acl-config-server.json | 237 +++++ .../restapi/responses/acl-tenant-node.json | 179 ++++ .../provision/restapi/responses/active-nodes.json | 15 + .../provision/restapi/responses/application1.json | 56 ++ .../restapi/responses/application2-nodes.json | 6 + .../provision/restapi/responses/application2.json | 32 + .../provision/restapi/responses/applications.json | 20 + .../responses/capacity-hostremoval-impossible.json | 20 + .../responses/capacity-hostremoval-possible.json | 20 + .../provision/restapi/responses/capacity-zone.json | 46 + .../hosted/provision/restapi/responses/cfg1.json | 34 + .../hosted/provision/restapi/responses/cfg2.json | 34 + .../provision/restapi/responses/child-nodes.json | 5 + .../provision/restapi/responses/content-nodes.json | 8 + .../provision/restapi/responses/controller1.json | 28 + .../restapi/responses/docker-container1.json | 51 + .../docker-node1-os-upgrade-complete.json | 73 ++ .../restapi/responses/docker-node1-os-upgrade.json | 68 ++ .../provision/restapi/responses/docker-node1.json | 66 ++ .../provision/restapi/responses/docker-node2.json | 66 ++ .../provision/restapi/responses/docker-node3.json | 66 ++ .../provision/restapi/responses/docker-node4.json | 66 ++ .../provision/restapi/responses/docker-node5.json | 66 ++ .../responses/dockerhost1-with-firmware-data.json | 73 ++ .../hosted/provision/restapi/responses/flags1.json | 10 + .../hosted/provision/restapi/responses/flags2.json | 15 + .../restapi/responses/load-balancers-single.json | 34 + .../restapi/responses/load-balancers.json | 64 ++ .../provision/restapi/responses/maintenance.json | 64 ++ .../hosted/provision/restapi/responses/node1.json | 56 ++ .../hosted/provision/restapi/responses/node10.json | 59 ++ .../hosted/provision/restapi/responses/node11.json | 29 + .../hosted/provision/restapi/responses/node13.json | 56 ++ .../hosted/provision/restapi/responses/node14.json | 56 ++ .../hosted/provision/restapi/responses/node2.json | 61 ++ .../hosted/provision/restapi/responses/node3.json | 33 + .../restapi/responses/node4-after-changes.json | 81 ++ .../hosted/provision/restapi/responses/node4.json | 64 ++ .../restapi/responses/node5-after-changes.json | 39 + .../hosted/provision/restapi/responses/node5.json | 41 + .../hosted/provision/restapi/responses/node55.json | 33 + .../restapi/responses/node6-after-changes.json | 60 ++ .../restapi/responses/node6-reports-2.json | 78 ++ .../restapi/responses/node6-reports-3.json | 66 ++ .../provision/restapi/responses/node6-reports.json | 81 ++ .../hosted/provision/restapi/responses/node6.json | 61 ++ .../hosted/provision/restapi/responses/node7.json | 28 + .../hosted/provision/restapi/responses/node8.json | 28 + .../hosted/provision/restapi/responses/node9.json | 29 + .../restapi/responses/nodes-recursive.json | 23 + .../hosted/provision/restapi/responses/nodes.json | 61 ++ .../provision/restapi/responses/parent1.json | 33 + .../provision/restapi/responses/parent2.json | 30 + .../hosted/provision/restapi/responses/root.json | 22 + .../restapi/responses/states-recursive.json | 68 ++ .../hosted/provision/restapi/responses/states.json | 31 + .../provision/restapi/v2/NodeSerializerTest.java | 30 - .../hosted/provision/restapi/v2/RestApiTest.java | 1044 -------------------- .../restapi/v2/responses/acl-config-server.json | 237 ----- .../restapi/v2/responses/acl-tenant-node.json | 179 ---- .../restapi/v2/responses/active-nodes.json | 15 - .../restapi/v2/responses/application1.json | 56 -- .../restapi/v2/responses/application2-nodes.json | 6 - .../restapi/v2/responses/application2.json | 32 - .../restapi/v2/responses/applications.json | 20 - .../responses/capacity-hostremoval-impossible.json | 20 - .../responses/capacity-hostremoval-possible.json | 20 - .../restapi/v2/responses/capacity-zone.json | 46 - .../provision/restapi/v2/responses/cfg1.json | 34 - .../provision/restapi/v2/responses/cfg2.json | 34 - .../restapi/v2/responses/child-nodes.json | 5 - .../restapi/v2/responses/content-nodes.json | 8 - .../restapi/v2/responses/controller1.json | 28 - .../restapi/v2/responses/docker-container1.json | 51 - .../docker-node1-os-upgrade-complete.json | 73 -- .../v2/responses/docker-node1-os-upgrade.json | 68 -- .../restapi/v2/responses/docker-node1.json | 66 -- .../restapi/v2/responses/docker-node2.json | 66 -- .../restapi/v2/responses/docker-node3.json | 66 -- .../restapi/v2/responses/docker-node4.json | 66 -- .../restapi/v2/responses/docker-node5.json | 66 -- .../responses/dockerhost1-with-firmware-data.json | 73 -- .../provision/restapi/v2/responses/flags1.json | 10 - .../provision/restapi/v2/responses/flags2.json | 15 - .../v2/responses/load-balancers-single.json | 34 - .../restapi/v2/responses/load-balancers.json | 64 -- .../restapi/v2/responses/maintenance.json | 64 -- .../provision/restapi/v2/responses/node1.json | 56 -- .../provision/restapi/v2/responses/node10.json | 59 -- .../provision/restapi/v2/responses/node11.json | 29 - .../provision/restapi/v2/responses/node13.json | 56 -- .../provision/restapi/v2/responses/node14.json | 56 -- .../provision/restapi/v2/responses/node2.json | 61 -- .../provision/restapi/v2/responses/node3.json | 33 - .../restapi/v2/responses/node4-after-changes.json | 81 -- .../provision/restapi/v2/responses/node4.json | 64 -- .../restapi/v2/responses/node5-after-changes.json | 39 - .../provision/restapi/v2/responses/node5.json | 41 - .../provision/restapi/v2/responses/node55.json | 33 - .../restapi/v2/responses/node6-after-changes.json | 60 -- .../restapi/v2/responses/node6-reports-2.json | 78 -- .../restapi/v2/responses/node6-reports-3.json | 66 -- .../restapi/v2/responses/node6-reports.json | 81 -- .../provision/restapi/v2/responses/node6.json | 61 -- .../provision/restapi/v2/responses/node7.json | 28 - .../provision/restapi/v2/responses/node8.json | 28 - .../provision/restapi/v2/responses/node9.json | 29 - .../restapi/v2/responses/nodes-recursive.json | 23 - .../provision/restapi/v2/responses/nodes.json | 61 -- .../provision/restapi/v2/responses/parent1.json | 33 - .../provision/restapi/v2/responses/parent2.json | 30 - .../provision/restapi/v2/responses/root.json | 22 - .../restapi/v2/responses/states-recursive.json | 68 -- .../provision/restapi/v2/responses/states.json | 31 - 148 files changed, 5609 insertions(+), 5542 deletions(-) create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java create mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ApplicationSerializer.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java delete mode 100644 node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializerTest.java create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-tenant-node.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/applications.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-impossible.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-possible.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/child-nodes.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags1.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags2.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-after-changes.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent1.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/root.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application1.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/applications.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-impossible.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-possible.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-zone.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers-single.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json delete mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 4e0fd95384c..2e40596bcf0 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; +import com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.After; import org.junit.Before; @@ -52,7 +53,7 @@ public class RealNodeRepositoryTest { * Starts NodeRepository with * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors} * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository} - * {@link com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler} + * {@link NodesV2ApiHandler} * These classes define some test data that is used in these tests. */ @Before diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml index 186f052a274..90e98d11ac0 100644 --- a/node-repository/src/main/config/node-repository.xml +++ b/node-repository/src/main/config/node-repository.xml @@ -9,11 +9,11 @@ - + http://*/nodes/v2/* - + http://*/loadbalancers/v1/* diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 8b019d54d49..a4e5255ad09 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -38,7 +38,7 @@ import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import com.yahoo.vespa.hosted.provision.provisioning.DockerImages; import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks; -import com.yahoo.vespa.hosted.provision.restapi.v2.NotFoundException; +import com.yahoo.vespa.hosted.provision.restapi.NotFoundException; import java.time.Clock; import java.time.Duration; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java new file mode 100644 index 00000000000..90081ecb604 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -0,0 +1,47 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Cluster; + +import java.net.URI; +import java.util.Collection; + +/** + * Serializes application information for nodes/v2/application responses + */ +public class ApplicationSerializer { + + public static Slime toSlime(Application application, URI applicationUri) { + Slime slime = new Slime(); + toSlime(application, slime.setObject(), applicationUri); + return slime; + } + + private static void toSlime(Application application, Cursor object, URI applicationUri) { + object.setString("url", applicationUri.toString()); + object.setString("id", application.id().toFullString()); + clustersToSlime(application.clusters().values(), object.setObject("clusters")); + } + + private static void clustersToSlime(Collection clusters, Cursor clustersObject) { + clusters.forEach(cluster -> toSlime(cluster, clustersObject.setObject(cluster.id().value()))); + } + + private static void toSlime(Cluster cluster, Cursor clusterObject) { + toSlime(cluster.minResources(), clusterObject.setObject("min")); + toSlime(cluster.maxResources(), clusterObject.setObject("max")); + cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject("suggested"))); + cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject("target"))); + } + + private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { + clusterResourcesObject.setLong("nodes", resources.nodes()); + clusterResourcesObject.setLong("groups", resources.groups()); + NodeResourcesSerializer.toSlime(resources.nodeResources(), clusterResourcesObject.setObject("resources")); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java new file mode 100644 index 00000000000..12a29707303 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java @@ -0,0 +1,164 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.maintenance.CapacityChecker; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +/** + * @author mgimle + */ +public class HostCapacityResponse extends HttpResponse { + private final StringBuilder text; + private final Slime slime; + private final CapacityChecker capacityChecker; + private final boolean json; + + public HostCapacityResponse(NodeRepository nodeRepository, HttpRequest request) { + super(200); + capacityChecker = new CapacityChecker(nodeRepository); + + json = request.getBooleanProperty("json"); + String hostsJson = request.getProperty("hosts"); + + text = new StringBuilder(); + slime = new Slime(); + Cursor root = slime.setObject(); + + if (hostsJson != null) { + List hosts = parseHostList(hostsJson); + hostRemovalResponse(root, hosts); + } else { + zoneFailureReponse(root); + } + } + + private List parseHostList(String hosts) { + List hostNames = Arrays.asList(hosts.split(",")); + try { + return capacityChecker.nodesFromHostnames(hostNames); + } catch (IllegalArgumentException e) { + throw new NotFoundException(e.getMessage()); + } + } + + private void hostRemovalResponse(Cursor root, List hosts) { + var failure = capacityChecker.findHostRemovalFailure(hosts); + if (failure.isPresent() && failure.get().failureReason.allocationFailures.size() == 0) { + root.setBool("removalPossible", false); + error(root, "Removing all hosts is trivially impossible."); + } else { + if (json) hostLossPossibleToSlime(root, failure, hosts); + else hostLossPossibleToText(failure, hosts); + } + } + + private void zoneFailureReponse(Cursor root) { + var failurePath = capacityChecker.worstCaseHostLossLeadingToFailure(); + if (failurePath.isPresent()) { + if (json) zoneFailurePathToSlime(root, failurePath.get()); + else zoneFailurePathToText(failurePath.get()); + } else { + error(root, "Node repository contained no hosts."); + } + } + + private void error(Cursor root, String errorMessage) { + if (json) root.setString("error", errorMessage); + else text.append(errorMessage); + } + + private void hostLossPossibleToText(Optional failure, List hostsToRemove) { + text.append(String.format("Attempting to remove %d hosts: ", hostsToRemove.size())); + CapacityChecker.AllocationHistory history = capacityChecker.allocationHistory; + if (failure.isEmpty()) { + text.append("OK\n\n"); + text.append(history); + if (history.oldParents().size() != hostsToRemove.size()) { + long emptyHostCount = hostsToRemove.size() - history.oldParents().size(); + text.append(String.format("\nTrivially removed %d empty host%s.", emptyHostCount, emptyHostCount > 1 ? "s" : "")); + } + } else { + text.append("FAILURE\n\n"); + text.append(history).append("\n"); + text.append(failure.get().failureReason).append("\n\n"); + } + } + + private void zoneFailurePathToText(CapacityChecker.HostFailurePath failurePath) { + text.append(String.format("Found %d hosts. Failure upon trying to remove %d hosts:\n\n", + capacityChecker.getHosts().size(), + failurePath.hostsCausingFailure.size())); + text.append(capacityChecker.allocationHistory).append("\n"); + text.append(failurePath.failureReason); + } + + private void hostLossPossibleToSlime(Cursor root, Optional failure, List hostsToRemove) { + var hosts = root.setArray("hostsToRemove"); + hostsToRemove.forEach(h -> hosts.addString(h.hostname())); + CapacityChecker.AllocationHistory history = capacityChecker.allocationHistory; + root.setBool("removalPossible", failure.isEmpty()); + var arr = root.setArray("history"); + for (var entry : history.historyEntries) { + var object = arr.addObject(); + object.setString("tenant", entry.tenant.hostname()); + if (entry.newParent != null) { + object.setString("newParent", entry.newParent.hostname()); + } + object.setLong("eligibleParents", entry.eligibleParents); + } + } + + private void zoneFailurePathToSlime(Cursor object, CapacityChecker.HostFailurePath failurePath) { + object.setLong("totalHosts", capacityChecker.getHosts().size()); + object.setLong("couldLoseHosts", failurePath.hostsCausingFailure.size()); + failurePath.failureReason.host.ifPresent(host -> + object.setString("failedTenantParent", host.hostname()) + ); + failurePath.failureReason.tenant.ifPresent(tenant -> { + object.setString("failedTenant", tenant.hostname()); + object.setString("failedTenantResources", tenant.flavor().resources().toString()); + tenant.allocation().ifPresent(allocation -> + object.setString("failedTenantAllocation", allocation.toString()) + ); + var explanation = object.setObject("hostCandidateRejectionReasons"); + allocationFailureReasonListToSlime(explanation.setObject("singularReasonFailures"), + failurePath.failureReason.allocationFailures.singularReasonFailures()); + allocationFailureReasonListToSlime(explanation.setObject("totalFailures"), + failurePath.failureReason.allocationFailures); + }); + var details = object.setObject("details"); + hostLossPossibleToSlime(details, Optional.of(failurePath), failurePath.hostsCausingFailure); + } + + private void allocationFailureReasonListToSlime(Cursor root, CapacityChecker.AllocationFailureReasonList allocationFailureReasonList) { + root.setLong("insufficientVcpu", allocationFailureReasonList.insufficientVcpu()); + root.setLong("insufficientMemoryGb", allocationFailureReasonList.insufficientMemoryGb()); + root.setLong("insufficientDiskGb", allocationFailureReasonList.insufficientDiskGb()); + root.setLong("incompatibleDiskSpeed", allocationFailureReasonList.incompatibleDiskSpeed()); + root.setLong("insufficientAvailableIps", allocationFailureReasonList.insufficientAvailableIps()); + root.setLong("violatesParentHostPolicy", allocationFailureReasonList.violatesParentHostPolicy()); + } + + @Override + public void render(OutputStream stream) throws IOException { + if (json) new JsonFormat(true).encode(stream, slime); + else stream.write(text.toString().getBytes()); + } + + @Override + public String getContentType() { + return json ? "application/json" : "text/plain"; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java new file mode 100644 index 00000000000..3b6de41f142 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java @@ -0,0 +1,42 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.maintenance.JobControl; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.TreeSet; + +/** A response containing maintenance job status */ +public class JobsResponse extends HttpResponse { + + private final JobControl jobControl; + + public JobsResponse(JobControl jobControl) { + super(200); + this.jobControl = jobControl; + } + + @Override + public void render(OutputStream stream) throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor jobArray = root.setArray("jobs"); + for (String jobName : jobControl.jobs()) + jobArray.addObject().setString("name", jobName); + + Cursor inactiveArray = root.setArray("inactive"); + for (String jobName : new TreeSet<>(jobControl.inactiveJobs())) + inactiveArray.addString(jobName); + + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { return "application/json"; } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java new file mode 100644 index 00000000000..092426c75d4 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -0,0 +1,89 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList; +import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.Optional; + +/** + * @author mpolden + */ +public class LoadBalancersResponse extends HttpResponse { + + private final NodeRepository nodeRepository; + private final HttpRequest request; + + public LoadBalancersResponse(HttpRequest request, NodeRepository nodeRepository) { + super(200); + this.request = request; + this.nodeRepository = nodeRepository; + } + + private Optional application() { + return Optional.ofNullable(request.getProperty("application")) + .map(ApplicationFilter::toApplicationId); + } + + private List loadBalancers() { + LoadBalancerList loadBalancers; + var application = application(); + if (application.isPresent()) { + loadBalancers = nodeRepository.loadBalancers(application.get()); + } else { + loadBalancers = nodeRepository.loadBalancers(); + } + return loadBalancers.asList(); + } + + @Override + public String getContentType() { return "application/json"; } + + @Override + public void render(OutputStream stream) throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor loadBalancerArray = root.setArray("loadBalancers"); + + loadBalancers().forEach(lb -> { + Cursor lbObject = loadBalancerArray.addObject(); + lbObject.setString("id", lb.id().serializedForm()); + lbObject.setString("state", lb.state().name()); + lbObject.setLong("changedAt", lb.changedAt().toEpochMilli()); + lbObject.setString("application", lb.id().application().application().value()); + lbObject.setString("tenant", lb.id().application().tenant().value()); + lbObject.setString("instance", lb.id().application().instance().value()); + lbObject.setString("cluster", lb.id().cluster().value()); + lbObject.setString("hostname", lb.instance().hostname().value()); + lb.instance().dnsZone().ifPresent(dnsZone -> lbObject.setString("dnsZone", dnsZone.id())); + + Cursor networkArray = lbObject.setArray("networks"); + lb.instance().networks().forEach(networkArray::addString); + + Cursor portArray = lbObject.setArray("ports"); + lb.instance().ports().forEach(portArray::addLong); + + Cursor realArray = lbObject.setArray("reals"); + lb.instance().reals().forEach(real -> { + Cursor realObject = realArray.addObject(); + realObject.setString("hostname", real.hostname().value()); + realObject.setString("ipAddress", real.ipAddress()); + realObject.setLong("port", real.port()); + }); + }); + + new JsonFormat(true).encode(stream, slime); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java new file mode 100644 index 00000000000..f81e3240397 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java @@ -0,0 +1,53 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.vespa.hosted.provision.NoSuchNodeException; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.yolean.Exceptions; + +import javax.inject.Inject; +import java.util.logging.Level; + +/** + * @author mpolden + */ +public class LoadBalancersV1ApiHandler extends LoggingRequestHandler { + + private final NodeRepository nodeRepository; + + @Inject + public LoadBalancersV1ApiHandler(LoggingRequestHandler.Context parentCtx, NodeRepository nodeRepository) { + super(parentCtx); + this.nodeRepository = nodeRepository; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: return handleGET(request); + default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + } + } + catch (NotFoundException | NoSuchNodeException e) { + return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); + } + catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse handleGET(HttpRequest request) { + String path = request.getUri().getPath(); + if (path.equals("/loadbalancers/v1/")) return new LoadBalancersResponse(request, nodeRepository); + throw new NotFoundException("Nothing at path '" + path + "'"); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java new file mode 100644 index 00000000000..a7577392fe2 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java @@ -0,0 +1,96 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.NodeAcl; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.Set; + +/** + * @author mpolden + */ +public class NodeAclResponse extends HttpResponse { + + private static final String CHILDREN_REQUEST_PROPERTY = "children"; + + private final NodeRepository nodeRepository; + private final Slime slime; + private final boolean aclsForChildren; + + public NodeAclResponse(HttpRequest request, NodeRepository nodeRepository) { + super(200); + this.nodeRepository = nodeRepository; + this.slime = new Slime(); + this.aclsForChildren = request.getBooleanProperty(CHILDREN_REQUEST_PROPERTY); + + Cursor root = slime.setObject(); + String hostname = baseName(request.getUri().getPath()); + toSlime(hostname, root); + } + + private void toSlime(String hostname, Cursor object) { + Node node = nodeRepository.getNode(hostname) + .orElseThrow(() -> new NotFoundException("No node with hostname '" + hostname + "'")); + + List acls = nodeRepository.getNodeAcls(node, aclsForChildren); + + Cursor trustedNodesArray = object.setArray("trustedNodes"); + acls.forEach(nodeAcl -> toSlime(nodeAcl, trustedNodesArray)); + + Cursor trustedNetworksArray = object.setArray("trustedNetworks"); + acls.forEach(nodeAcl -> toSlime(nodeAcl.trustedNetworks(), nodeAcl.node(), trustedNetworksArray)); + + Cursor trustedPortsArray = object.setArray("trustedPorts"); + acls.forEach(nodeAcl -> toSlime(nodeAcl.trustedPorts(), nodeAcl, trustedPortsArray)); + } + + private void toSlime(NodeAcl nodeAcl, Cursor array) { + nodeAcl.trustedNodes().forEach(node -> node.ipAddresses().forEach(ipAddress -> { + Cursor object = array.addObject(); + object.setString("hostname", node.hostname()); + object.setString("type", node.type().name()); + object.setString("ipAddress", ipAddress); + object.setString("trustedBy", nodeAcl.node().hostname()); + })); + } + + private void toSlime(Set trustedNetworks, Node trustedby, Cursor array) { + trustedNetworks.forEach(network -> { + Cursor object = array.addObject(); + object.setString("network", network); + object.setString("trustedBy", trustedby.hostname()); + }); + } + + private void toSlime(Set trustedPorts, NodeAcl trustedBy, Cursor array) { + trustedPorts.forEach(port -> { + Cursor object = array.addObject(); + object.setLong("port", port); + object.setString("trustedBy", trustedBy.node().hostname()); + }); + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { + return "application/json"; + } + + private static String baseName(String path) { + return new File(path).getName(); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java new file mode 100644 index 00000000000..8b5639cc514 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -0,0 +1,270 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; +import com.yahoo.io.IOUtils; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Type; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.provision.LockedNodeList; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.IP; +import com.yahoo.vespa.hosted.provision.node.Report; +import com.yahoo.vespa.hosted.provision.node.Reports; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.time.Clock; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; +import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; +import static com.yahoo.config.provision.NodeResources.StorageType.local; +import static com.yahoo.config.provision.NodeResources.StorageType.remote; + +/** + * A class which can take a partial JSON node/v2 node JSON structure and apply it to a node object. + * This is a one-time use object. + * + * @author bratseth + */ +public class NodePatcher { + + private static final String WANT_TO_RETIRE = "wantToRetire"; + + private final NodeFlavors nodeFlavors; + private final Inspector inspector; + private final Supplier nodes; + private final Clock clock; + + private Node node; + + public NodePatcher(NodeFlavors nodeFlavors, InputStream json, Node node, Supplier nodes, Clock clock) { + this.nodeFlavors = nodeFlavors; + this.node = node; + this.nodes = nodes; + this.clock = clock; + try { + this.inspector = SlimeUtils.jsonToSlime(IOUtils.readBytes(json, 1000 * 1000)).get(); + } catch (IOException e) { + throw new UncheckedIOException("Error reading request body", e); + } + } + + /** + * Apply the json to the node and return all nodes affected by the patch. + * More than 1 node may be affected if e.g. the node is a Docker host, which may have + * children that must be updated in a consistent manner. + */ + public List apply() { + List patchedNodes = new ArrayList<>(); + inspector.traverse((String name, Inspector value) -> { + try { + node = applyField(node, name, value); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Could not set field '" + name + "'", e); + } + + try { + patchedNodes.addAll(applyFieldRecursive(name, value)); + } catch (IllegalArgumentException e) { + // Non recursive field, ignore + } + } ); + patchedNodes.add(node); + + return patchedNodes; + } + + private List applyFieldRecursive(String name, Inspector value) { + switch (name) { + case WANT_TO_RETIRE: + List childNodes = node.type().isDockerHost() ? nodes.get().childrenOf(node).asList() : List.of(); + return childNodes.stream() + .map(child -> applyField(child, name, value)) + .collect(Collectors.toList()); + + default : + throw new IllegalArgumentException("Field " + name + " is not recursive"); + } + } + + private Node applyField(Node node, String name, Inspector value) { + switch (name) { + case "currentRebootGeneration" : + return node.withCurrentRebootGeneration(asLong(value), clock.instant()); + case "currentRestartGeneration" : + return patchCurrentRestartGeneration(asLong(value)); + case "currentDockerImage" : + if (node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) + throw new IllegalArgumentException("Docker image can only be set for docker containers"); + return node.with(node.status().withDockerImage(DockerImage.fromString(asString(value)))); + case "vespaVersion" : + case "currentVespaVersion" : + return node.with(node.status().withVespaVersion(Version.fromString(asString(value)))); + case "currentOsVersion" : + return node.withCurrentOsVersion(Version.fromString(asString(value)), clock.instant()); + case "currentFirmwareCheck": + return node.withFirmwareVerifiedAt(Instant.ofEpochMilli(asLong(value))); + case "failCount" : + return node.with(node.status().withFailCount(asLong(value).intValue())); + case "flavor" : + return node.with(nodeFlavors.getFlavorOrThrow(asString(value))); + case "parentHostname" : + return node.withParentHostname(asString(value)); + case "ipAddresses" : + return IP.Config.verify(node.with(node.ipConfig().with(asStringSet(value))), nodes.get()); + case "additionalIpAddresses" : + return IP.Config.verify(node.with(node.ipConfig().with(IP.Pool.of(asStringSet(value)))), nodes.get()); + case WANT_TO_RETIRE : + return node.withWantToRetire(asBoolean(value), Agent.operator, clock.instant()); + case "wantToDeprovision" : + if (node.type() != NodeType.host && asBoolean(value)) + throw new IllegalArgumentException("wantToDeprovision can only be set for hosts"); + return node.with(node.status().withWantToDeprovision(asBoolean(value))); + case "reports" : + return nodeWithPatchedReports(node, value); + case "openStackId" : + return node.withOpenStackId(asString(value)); + case "diskGb": + case "minDiskAvailableGb": + return node.with(node.flavor().with(node.flavor().resources().withDiskGb(value.asDouble()))); + case "memoryGb": + case "minMainMemoryAvailableGb": + return node.with(node.flavor().with(node.flavor().resources().withMemoryGb(value.asDouble()))); + case "vcpu": + case "minCpuCores": + return node.with(node.flavor().with(node.flavor().resources().withVcpu(value.asDouble()))); + case "fastDisk": + return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? fast : slow))); + case "remoteStorage": + return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? remote : local))); + case "bandwidthGbps": + return node.with(node.flavor().with(node.flavor().resources().withBandwidthGbps(value.asDouble()))); + case "modelName": + return value.type() == Type.NIX ? node.withoutModelName() : node.withModelName(asString(value)); + case "requiredDiskSpeed": + return patchRequiredDiskSpeed(asString(value)); + case "reservedTo": + return value.type() == Type.NIX ? node.withoutReservedTo() : node.withReservedTo(TenantName.from(value.asString())); + default : + throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); + } + } + + private Node nodeWithPatchedReports(Node node, Inspector reportsInspector) { + Node patchedNode; + // "reports": null clears the reports + if (reportsInspector.type() == Type.NIX) { + patchedNode = node.with(new Reports()); + } else { + var reportsBuilder = new Reports.Builder(node.reports()); + reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> { + if (reportInspector.type() == Type.NIX) { + // ... "reports": { "reportId": null } clears the report "reportId" + reportsBuilder.clearReport(reportId); + } else { + // ... "reports": { "reportId": {...} } overrides the whole report "reportId" + reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector)); + } + }); + patchedNode = node.with(reportsBuilder.build()); + } + + boolean hadHardFailReports = node.reports().getReports().stream() + .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL); + boolean hasHardFailReports = patchedNode.reports().getReports().stream() + .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL); + + // If this patch resulted in going from not having HARD_FAIL report to having one, or vice versa + if (hadHardFailReports != hasHardFailReports) { + // Do not automatically change wantToDeprovision when + // 1. Transitioning to having a HARD_FAIL report and being in state failed: + // To allow operators manually unset before the host is parked and deleted. + // 2. When in parked state: Deletion is imminent, possibly already underway + if ((hasHardFailReports && node.state() == Node.State.failed) || node.state() == Node.State.parked) + return patchedNode; + + patchedNode = patchedNode.with(patchedNode.status().withWantToDeprovision(hasHardFailReports)); + } + + return patchedNode; + } + + private Set asStringSet(Inspector field) { + if ( ! field.type().equals(Type.ARRAY)) + throw new IllegalArgumentException("Expected an ARRAY value, got a " + field.type()); + + TreeSet strings = new TreeSet<>(); + for (int i = 0; i < field.entries(); i++) { + Inspector entry = field.entry(i); + if ( ! entry.type().equals(Type.STRING)) + throw new IllegalArgumentException("Expected a STRING value, got a " + entry.type()); + strings.add(entry.asString()); + } + + return strings; + } + + private Node patchRequiredDiskSpeed(String value) { + Optional allocation = node.allocation(); + if (allocation.isPresent()) + return node.with(allocation.get().withRequestedResources( + allocation.get().requestedResources().with(NodeResources.DiskSpeed.valueOf(value)))); + else + throw new IllegalArgumentException("Node is not allocated"); + } + + private Node patchCurrentRestartGeneration(Long value) { + Optional allocation = node.allocation(); + if (allocation.isPresent()) + return node.with(allocation.get().withRestart(allocation.get().restartGeneration().withCurrent(value))); + else + throw new IllegalArgumentException("Node is not allocated"); + } + + private Long asLong(Inspector field) { + if ( ! field.type().equals(Type.LONG)) + throw new IllegalArgumentException("Expected a LONG value, got a " + field.type()); + return field.asLong(); + } + + private String asString(Inspector field) { + if ( ! field.type().equals(Type.STRING)) + throw new IllegalArgumentException("Expected a STRING value, got a " + field.type()); + return field.asString(); + } + + private Optional asOptionalString(Inspector field) { + return field.type().equals(Type.NIX) ? Optional.empty() : Optional.of(asString(field)); + } + + // Allows us to clear optional flags by passing "null" as slime does not have an empty (but present) representation + private Optional removeQuotedNulls(Optional value) { + return value.filter(v -> !v.equals("null")); + } + + private boolean asBoolean(Inspector field) { + if ( ! field.type().equals(Type.BOOL)) + throw new IllegalArgumentException("Expected a BOOL value, got a " + field.type()); + return field.asBool(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java new file mode 100644 index 00000000000..625ee89434e --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.config.provision.NodeResources; +import com.yahoo.slime.Cursor; + +/** + * @author bratseth + */ +public class NodeResourcesSerializer { + + static void toSlime(NodeResources resources, Cursor object) { + object.setDouble("vcpu", resources.vcpu()); + object.setDouble("memoryGb", resources.memoryGb()); + object.setDouble("diskGb", resources.diskGb()); + object.setDouble("bandwidthGbps", resources.bandwidthGbps()); + object.setString("diskSpeed", toString(resources.diskSpeed())); + object.setString("storageType", toString(resources.storageType())); + } + + public static NodeResources.DiskSpeed diskSpeedFrom(String diskSpeed) { + switch (diskSpeed) { + case "fast": return NodeResources.DiskSpeed.fast; + case "slow": return NodeResources.DiskSpeed.slow; + case "any" : return NodeResources.DiskSpeed.any; + default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed + "'"); + } + } + + private static String toString(NodeResources.DiskSpeed diskSpeed) { + switch (diskSpeed) { + case fast : return "fast"; + case slow : return "slow"; + case any : return "any"; + default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed.name() + "'"); + } + } + + public static NodeResources.StorageType storageTypeFrom(String storageType) { + switch (storageType) { + case "local" : return NodeResources.StorageType.local; + case "remote": return NodeResources.StorageType.remote; + case "any" : return NodeResources.StorageType.any; + default: throw new IllegalArgumentException("Unknown storage type '" + storageType + "'"); + } + } + + private static String toString(NodeResources.StorageType storageType) { + switch (storageType) { + case remote : return "remote"; + case local : return "local"; + case any : return "any"; + default: throw new IllegalArgumentException("Unknown storage type '" + storageType.name() + "'"); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java new file mode 100644 index 00000000000..bd65894101c --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java @@ -0,0 +1,76 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.slime.Cursor; +import com.yahoo.vespa.hosted.provision.Node; + +/** + * Serializer for node wire format. Used to serialize node objects sent over the wire (HTTP). + * + * @author mpolden + */ +public class NodeSerializer { + + public static Node.State stateFrom(String state) { + switch (state) { + case "active": return Node.State.active; + case "dirty": return Node.State.dirty; + case "failed": return Node.State.failed; + case "inactive": return Node.State.inactive; + case "parked": return Node.State.parked; + case "provisioned": return Node.State.provisioned; + case "ready": return Node.State.ready; + case "reserved": return Node.State.reserved; + case "deprovisioned": return Node.State.deprovisioned; + default: throw new IllegalArgumentException("Unknown node state '" + state + "'"); + } + } + + public static String toString(Node.State state) { + switch (state) { + case active: return "active"; + case dirty: return "dirty"; + case failed: return "failed"; + case inactive: return "inactive"; + case parked: return "parked"; + case provisioned: return "provisioned"; + case ready: return "ready"; + case reserved: return "reserved"; + case deprovisioned: return "deprovisioned"; + default: throw new IllegalArgumentException("Unknown node state '" + state + "'"); + } + } + + public static NodeType typeFrom(String nodeType) { + switch (nodeType) { + case "tenant": return NodeType.tenant; + case "host": return NodeType.host; + case "proxy": return NodeType.proxy; + case "proxyhost": return NodeType.proxyhost; + case "config": return NodeType.config; + case "confighost": return NodeType.confighost; + case "controller": return NodeType.controller; + case "controllerhost": return NodeType.controllerhost; + case "devhost": return NodeType.devhost; + default: throw new IllegalArgumentException("Unknown node type '" + nodeType + "'"); + } + } + + public static String toString(NodeType type) { + switch (type) { + case tenant: return "tenant"; + case host: return "host"; + case proxy: return "proxy"; + case proxyhost: return "proxyhost"; + case config: return "config"; + case confighost: return "confighost"; + case controller: return "controller"; + case controllerhost: return "controllerhost"; + case devhost: return "devhost"; + default: throw new IllegalArgumentException("Unknown node type '" + type.name() + "'"); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java new file mode 100644 index 00000000000..9d02d2907cc --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -0,0 +1,233 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.serialization.NetworkPortsSerializer; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; +import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.vespa.orchestrator.status.HostInfo; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +/** +* @author bratseth +*/ +class NodesResponse extends HttpResponse { + + /** The responses this can create */ + public enum ResponseType { nodeList, stateList, nodesInStateList, singleNode } + + /** The request url minus parameters, with a trailing slash added if missing */ + private final String parentUrl; + + /** The parent url of nodes */ + private final String nodeParentUrl; + + private final NodeFilter filter; + private final boolean recursive; + private final Function> orchestrator; + private final NodeRepository nodeRepository; + private final Slime slime; + + public NodesResponse(ResponseType responseType, HttpRequest request, + Orchestrator orchestrator, NodeRepository nodeRepository) { + super(200); + this.parentUrl = toParentUrl(request); + this.nodeParentUrl = toNodeParentUrl(request); + filter = NodesV2ApiHandler.toNodeFilter(request); + this.recursive = request.getBooleanProperty("recursive"); + this.orchestrator = orchestrator.getHostResolver(); + this.nodeRepository = nodeRepository; + + slime = new Slime(); + Cursor root = slime.setObject(); + switch (responseType) { + case nodeList: nodesToSlime(root); break; + case stateList : statesToSlime(root); break; + case nodesInStateList: nodesToSlime(NodeSerializer.stateFrom(lastElement(parentUrl)), root); break; + case singleNode : nodeToSlime(lastElement(parentUrl), root); break; + default: throw new IllegalArgumentException(); + } + } + + private String toParentUrl(HttpRequest request) { + URI uri = request.getUri(); + String parentUrl = uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + uri.getPath(); + if ( ! parentUrl.endsWith("/")) + parentUrl = parentUrl + "/"; + return parentUrl; + } + + private String toNodeParentUrl(HttpRequest request) { + URI uri = request.getUri(); + return uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + "/nodes/v2/node/"; + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { + return "application/json"; + } + + private void statesToSlime(Cursor root) { + Cursor states = root.setObject("states"); + for (Node.State state : Node.State.values()) + toSlime(state, states.setObject(NodeSerializer.toString(state))); + } + + private void toSlime(Node.State state, Cursor object) { + object.setString("url", parentUrl + NodeSerializer.toString(state)); + if (recursive) + nodesToSlime(state, object); + } + + /** Outputs the nodes in the given state to a node array */ + private void nodesToSlime(Node.State state, Cursor parentObject) { + Cursor nodeArray = parentObject.setArray("nodes"); + for (NodeType type : NodeType.values()) + toSlime(nodeRepository.getNodes(type, state), nodeArray); + } + + /** Outputs all the nodes to a node array */ + private void nodesToSlime(Cursor parentObject) { + Cursor nodeArray = parentObject.setArray("nodes"); + toSlime(nodeRepository.getNodes(), nodeArray); + } + + private void toSlime(List nodes, Cursor array) { + for (Node node : nodes) { + if ( ! filter.matches(node)) continue; + toSlime(node, recursive, array.addObject()); + } + } + + private void nodeToSlime(String hostname, Cursor object) { + Node node = nodeRepository.getNode(hostname).orElseThrow(() -> + new NotFoundException("No node with hostname '" + hostname + "'")); + toSlime(node, true, object); + } + + private void toSlime(Node node, boolean allFields, Cursor object) { + object.setString("url", nodeParentUrl + node.hostname()); + if ( ! allFields) return; + object.setString("id", node.hostname()); + object.setString("state", NodeSerializer.toString(node.state())); + object.setString("type", node.type().name()); + object.setString("hostname", node.hostname()); + object.setString("type", NodeSerializer.toString(node.type())); + if (node.parentHostname().isPresent()) { + object.setString("parentHostname", node.parentHostname().get()); + } + object.setString("openStackId", node.id()); + object.setString("flavor", node.flavor().name()); + node.reservedTo().ifPresent(reservedTo -> object.setString("reservedTo", reservedTo.value())); + if (node.flavor().isConfigured()) + object.setDouble("cpuCores", node.flavor().getMinCpuCores()); + NodeResourcesSerializer.toSlime(node.flavor().resources(), object.setObject("resources")); + if (node.flavor().cost() > 0) + object.setLong("cost", node.flavor().cost()); + object.setString("environment", node.flavor().getType().name()); + node.allocation().ifPresent(allocation -> { + toSlime(allocation.owner(), object.setObject("owner")); + toSlime(allocation.membership(), object.setObject("membership")); + object.setLong("restartGeneration", allocation.restartGeneration().wanted()); + object.setLong("currentRestartGeneration", allocation.restartGeneration().current()); + object.setString("wantedDockerImage", allocation.membership().cluster().dockerImage() + .orElseGet(() -> nodeRepository.dockerImage(node).withTag(allocation.membership().cluster().vespaVersion()).asString())); + object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); + NodeResourcesSerializer.toSlime(allocation.requestedResources(), object.setObject("requestedResources")); + allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); + orchestrator.apply(new HostName(node.hostname())) + .ifPresent(info -> { + object.setBool("allowedToBeDown", info.status().isSuspended()); + info.suspendedSince().ifPresent(since -> object.setLong("suspendedSinceMillis", since.toEpochMilli())); + }); + }); + object.setLong("rebootGeneration", node.status().reboot().wanted()); + object.setLong("currentRebootGeneration", node.status().reboot().current()); + node.status().osVersion().current().ifPresent(version -> object.setString("currentOsVersion", version.toFullString())); + node.status().osVersion().wanted().ifPresent(version -> object.setString("wantedOsVersion", version.toFullString())); + node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong("currentFirmwareCheck", instant.toEpochMilli())); + if (node.type().isDockerHost()) + nodeRepository.firmwareChecks().requiredAfter().ifPresent(after -> object.setLong("wantedFirmwareCheck", after.toEpochMilli())); + node.status().vespaVersion().ifPresent(version -> object.setString("vespaVersion", version.toFullString())); + currentDockerImage(node).ifPresent(dockerImage -> object.setString("currentDockerImage", dockerImage.asString())); + object.setLong("failCount", node.status().failCount()); + object.setBool("wantToRetire", node.status().wantToRetire()); + object.setBool("wantToDeprovision", node.status().wantToDeprovision()); + toSlime(node.history(), object.setArray("history")); + ipAddressesToSlime(node.ipAddresses(), object.setArray("ipAddresses")); + ipAddressesToSlime(node.ipAddressPool().asSet(), object.setArray("additionalIpAddresses")); + node.reports().toSlime(object, "reports"); + node.modelName().ifPresent(modelName -> object.setString("modelName", modelName)); + } + + private void toSlime(ApplicationId id, Cursor object) { + object.setString("tenant", id.tenant().value()); + object.setString("application", id.application().value()); + object.setString("instance", id.instance().value()); + } + + private void toSlime(ClusterMembership membership, Cursor object) { + object.setString("clustertype", membership.cluster().type().name()); + object.setString("clusterid", membership.cluster().id().value()); + object.setString("group", String.valueOf(membership.cluster().group().get().index())); + object.setLong("index", membership.index()); + object.setBool("retired", membership.retired()); + } + + private void toSlime(History history, Cursor array) { + for (History.Event event : history.events()) { + Cursor object = array.addObject(); + object.setString("event", event.type().name()); + object.setLong("at", event.at().toEpochMilli()); + object.setString("agent", event.agent().name()); + } + } + + // Hack: For non-docker nodes, return current docker image as default prefix + current Vespa version + // TODO: Remove current + wanted docker image from response for non-docker types + private Optional currentDockerImage(Node node) { + return node.status().dockerImage() + .or(() -> Optional.of(node) + .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) + .flatMap(n -> n.status().vespaVersion() + .map(version -> nodeRepository.dockerImage(n).withTag(version)))); + } + + private void ipAddressesToSlime(Set ipAddresses, Cursor array) { + ipAddresses.forEach(array::addString); + } + + private String lastElement(String path) { + if (path.endsWith("/")) + path = path.substring(0, path.length()-1); + int lastSlash = path.lastIndexOf("/"); + if (lastSlash < 0) return path; + return path.substring(lastSlash+1); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java new file mode 100644 index 00000000000..1e8bf622f72 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -0,0 +1,436 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.io.IOUtils; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.MessageResponse; +import com.yahoo.restapi.Path; +import com.yahoo.restapi.ResourceResponse; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; +import com.yahoo.vespa.hosted.provision.NoSuchNodeException; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.IP; +import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeOsVersionFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; +import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; +import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; +import com.yahoo.vespa.hosted.provision.restapi.NodesResponse.ResponseType; +import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.yolean.Exceptions; + +import javax.inject.Inject; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Level; +import java.util.stream.Collectors; + +import static com.yahoo.slime.SlimeUtils.optionalString; + +/** + * The implementation of the /nodes/v2 API. + * See RestApiTest for documentation. + * + * @author bratseth + */ +public class NodesV2ApiHandler extends LoggingRequestHandler { + + private final Orchestrator orchestrator; + private final NodeRepository nodeRepository; + private final NodeFlavors nodeFlavors; + + @Inject + public NodesV2ApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator, + NodeRepository nodeRepository, NodeFlavors flavors) { + super(parentCtx); + this.orchestrator = orchestrator; + this.nodeRepository = nodeRepository; + this.nodeFlavors = flavors; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: return handleGET(request); + case PUT: return handlePUT(request); + case POST: return isPatchOverride(request) ? handlePATCH(request) : handlePOST(request); + case DELETE: return handleDELETE(request); + case PATCH: return handlePATCH(request); + default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + } + } + catch (NotFoundException | NoSuchNodeException e) { + return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); + } + catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse handleGET(HttpRequest request) { + Path path = new Path(request.getUri()); + String pathS = request.getUri().getPath(); + if (pathS.equals( "/nodes/v2/")) return new ResourceResponse(request.getUri(), "state", "node", "command", "maintenance", "upgrade", "application"); + if (pathS.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository); + if (pathS.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository); + if (pathS.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, orchestrator, nodeRepository); + if (pathS.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, orchestrator, nodeRepository); + if (pathS.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); + if (pathS.equals( "/nodes/v2/command/")) return new ResourceResponse(request.getUri(), "restart", "reboot"); + if (pathS.equals( "/nodes/v2/maintenance/")) return new JobsResponse(nodeRepository.jobControl()); + if (pathS.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(nodeRepository.infrastructureVersions(), nodeRepository.osVersions(), nodeRepository.dockerImages()); + if (pathS.startsWith("/nodes/v2/capacity")) return new HostCapacityResponse(nodeRepository, request); + if (path.matches("/nodes/v2/application")) return applicationList(request.getUri()); + if (path.matches("/nodes/v2/application/{applicationId}")) return application(path.get("applicationId"), request.getUri()); + throw new NotFoundException("Nothing at " + path); + } + + private HttpResponse handlePUT(HttpRequest request) { + String path = request.getUri().getPath(); + // Check paths to disallow illegal state changes + if (path.startsWith("/nodes/v2/state/ready/") || + path.startsWith("/nodes/v2/state/availablefornewallocations/")) { + nodeRepository.markNodeAvailableForNewAllocation(lastElement(path), Agent.operator, "Readied through the nodes/v2 API"); + return new MessageResponse("Moved " + lastElement(path) + " to ready"); + } + else if (path.startsWith("/nodes/v2/state/failed/")) { + List failedNodes = nodeRepository.failRecursively(lastElement(path), Agent.operator, "Failed through the nodes/v2 API"); + return new MessageResponse("Moved " + hostnamesAsString(failedNodes) + " to failed"); + } + else if (path.startsWith("/nodes/v2/state/parked/")) { + List parkedNodes = nodeRepository.parkRecursively(lastElement(path), Agent.operator, "Parked through the nodes/v2 API"); + return new MessageResponse("Moved " + hostnamesAsString(parkedNodes) + " to parked"); + } + else if (path.startsWith("/nodes/v2/state/dirty/")) { + List dirtiedNodes = nodeRepository.dirtyRecursively(lastElement(path), Agent.operator, "Dirtied through the nodes/v2 API"); + return new MessageResponse("Moved " + hostnamesAsString(dirtiedNodes) + " to dirty"); + } + else if (path.startsWith("/nodes/v2/state/active/")) { + nodeRepository.reactivate(lastElement(path), Agent.operator, "Reactivated through nodes/v2 API"); + return new MessageResponse("Moved " + lastElement(path) + " to active"); + } + + throw new NotFoundException("Cannot put to path '" + path + "'"); + } + + private HttpResponse handlePATCH(HttpRequest request) { + String path = request.getUri().getPath(); + if (path.startsWith("/nodes/v2/node/")) { + // TODO: Node is fetched outside of lock, might change after getting lock + Node node = nodeFromRequest(request); + try (var lock = nodeRepository.lock(node)) { + var patchedNodes = new NodePatcher(nodeFlavors, request.getData(), node, () -> nodeRepository.list(lock), + nodeRepository.clock()).apply(); + nodeRepository.write(patchedNodes, lock); + } + return new MessageResponse("Updated " + node.hostname()); + } + else if (path.startsWith("/nodes/v2/upgrade/")) { + return setTargetVersions(request); + } + + throw new NotFoundException("Nothing at '" + path + "'"); + } + + private HttpResponse handlePOST(HttpRequest request) { + Path path = new Path(request.getUri()); + if (path.matches("/nodes/v2/command/restart")) { + int restartCount = nodeRepository.restart(toNodeFilter(request)).size(); + return new MessageResponse("Scheduled restart of " + restartCount + " matching nodes"); + } + if (path.matches("/nodes/v2/command/reboot")) { + int rebootCount = nodeRepository.reboot(toNodeFilter(request)).size(); + return new MessageResponse("Scheduled reboot of " + rebootCount + " matching nodes"); + } + if (path.matches("/nodes/v2/node")) { + int addedNodes = addNodes(request.getData()); + return new MessageResponse("Added " + addedNodes + " nodes to the provisioned state"); + } + if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), false); + if (path.matches("/nodes/v2/maintenance/run/{job}")) return runJob(path.get("job")); + if (path.matches("/nodes/v2/upgrade/firmware")) return requestFirmwareCheckResponse(); + + throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); + } + + private HttpResponse runJob(String job) { + nodeRepository.jobControl().run(job); + return new MessageResponse("Executed job '" + job + "'"); + } + + private HttpResponse handleDELETE(HttpRequest request) { + Path path = new Path(request.getUri()); + if (path.matches("/nodes/v2/node/{hostname}")) { + String hostname = path.get("hostname"); + List removedNodes = nodeRepository.removeRecursively(hostname); + return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", "))); + } + if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), true); + if (path.matches("/nodes/v2/upgrade/firmware")) return cancelFirmwareCheckResponse(); + + throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); + } + + private Node nodeFromRequest(HttpRequest request) { + String hostname = lastElement(request.getUri().getPath()); + return nodeRepository.getNode(hostname).orElseThrow(() -> + new NotFoundException("No node found with hostname " + hostname)); + } + + public int addNodes(InputStream jsonStream) { + List nodes = createNodesFromSlime(toSlime(jsonStream).get()); + return nodeRepository.addNodes(nodes, Agent.operator).size(); + } + + private Slime toSlime(InputStream jsonStream) { + try { + byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000); + return SlimeUtils.jsonToSlime(jsonBytes); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private List createNodesFromSlime(Inspector object) { + List nodes = new ArrayList<>(); + object.traverse((ArrayTraverser) (int i, Inspector item) -> nodes.add(createNode(item))); + return nodes; + } + + private Node createNode(Inspector inspector) { + Optional parentHostname = optionalString(inspector.field("parentHostname")); + Optional modelName = optionalString(inspector.field("modelName")); + Set ipAddresses = new HashSet<>(); + inspector.field("ipAddresses").traverse((ArrayTraverser) (i, item) -> ipAddresses.add(item.asString())); + Set ipAddressPool = new HashSet<>(); + inspector.field("additionalIpAddresses").traverse((ArrayTraverser) (i, item) -> ipAddressPool.add(item.asString())); + + return Node.create(inspector.field("openStackId").asString(), + new IP.Config(ipAddresses, ipAddressPool), + inspector.field("hostname").asString(), + parentHostname, + modelName, + flavorFromSlime(inspector), + reservedToFromSlime(inspector.field("reservedTo")), + nodeTypeFromSlime(inspector.field("type")) + ); + } + + private Flavor flavorFromSlime(Inspector inspector) { + Inspector flavorInspector = inspector.field("flavor"); + Inspector resourcesInspector = inspector.field("resources"); + if ( ! flavorInspector.valid()) { + return new Flavor(new NodeResources( + requiredField(resourcesInspector, "vcpu", Inspector::asDouble), + requiredField(resourcesInspector, "memoryGb", Inspector::asDouble), + requiredField(resourcesInspector, "diskGb", Inspector::asDouble), + requiredField(resourcesInspector, "bandwidthGbps", Inspector::asDouble), + optionalString(resourcesInspector.field("diskSpeed")).map(NodeResourcesSerializer::diskSpeedFrom).orElse(NodeResources.DiskSpeed.getDefault()), + optionalString(resourcesInspector.field("storageType")).map(NodeResourcesSerializer::storageTypeFrom).orElse(NodeResources.StorageType.getDefault()))); + } + + Flavor flavor = nodeFlavors.getFlavorOrThrow(flavorInspector.asString()); + if (resourcesInspector.valid()) { + if (resourcesInspector.field("vcpu").valid()) + flavor = flavor.with(flavor.resources().withVcpu(resourcesInspector.field("vcpu").asDouble())); + if (resourcesInspector.field("memoryGb").valid()) + flavor = flavor.with(flavor.resources().withMemoryGb(resourcesInspector.field("memoryGb").asDouble())); + if (resourcesInspector.field("diskGb").valid()) + flavor = flavor.with(flavor.resources().withDiskGb(resourcesInspector.field("diskGb").asDouble())); + if (resourcesInspector.field("bandwidthGbps").valid()) + flavor = flavor.with(flavor.resources().withBandwidthGbps(resourcesInspector.field("bandwidthGbps").asDouble())); + if (resourcesInspector.field("diskSpeed").valid()) + flavor = flavor.with(flavor.resources().with(NodeResourcesSerializer.diskSpeedFrom(resourcesInspector.field("diskSpeed").asString()))); + if (resourcesInspector.field("storageType").valid()) + flavor = flavor.with(flavor.resources().with(NodeResourcesSerializer.storageTypeFrom(resourcesInspector.field("storageType").asString()))); + } + return flavor; + } + + private static T requiredField(Inspector inspector, String fieldName, Function valueExtractor) { + Inspector field = inspector.field(fieldName); + if (!field.valid()) throw new IllegalArgumentException("Required field '" + fieldName + "' is missing"); + return valueExtractor.apply(field); + } + + private NodeType nodeTypeFromSlime(Inspector object) { + if (! object.valid()) return NodeType.tenant; // default + return NodeSerializer.typeFrom(object.asString()); + } + + private Optional reservedToFromSlime(Inspector object) { + if (! object.valid()) return Optional.empty(); + if ( object.type() != Type.STRING) + throw new IllegalArgumentException("Expected 'reservedTo' to be a string but is " + object); + return Optional.of(TenantName.from(object.asString())); + } + + public static NodeFilter toNodeFilter(HttpRequest request) { + NodeFilter filter = NodeHostFilter.from(HostFilter.from(request.getProperty("hostname"), + request.getProperty("flavor"), + request.getProperty("clusterType"), + request.getProperty("clusterId"))); + filter = ApplicationFilter.from(request.getProperty("application"), filter); + filter = StateFilter.from(request.getProperty("state"), filter); + filter = NodeTypeFilter.from(request.getProperty("type"), filter); + filter = ParentHostFilter.from(request.getProperty("parentHost"), filter); + filter = NodeOsVersionFilter.from(request.getProperty("osVersion"), filter); + return filter; + } + + private static String lastElement(String path) { + if (path.endsWith("/")) + path = path.substring(0, path.length()-1); + int lastSlash = path.lastIndexOf("/"); + if (lastSlash < 0) return path; + return path.substring(lastSlash + 1); + } + + private static boolean isPatchOverride(HttpRequest request) { + // Since Jersey's HttpUrlConnector does not support PATCH we support this by override this on POST requests. + String override = request.getHeader("X-HTTP-Method-Override"); + if (override != null) { + if (override.equals("PATCH")) { + return true; + } else { + String msg = String.format("Illegal X-HTTP-Method-Override header for POST request. Accepts 'PATCH' but got '%s'", override); + throw new IllegalArgumentException(msg); + } + } + return false; + } + + private MessageResponse setJobActive(String jobName, boolean active) { + if ( ! nodeRepository.jobControl().jobs().contains(jobName)) + throw new NotFoundException("No job named '" + jobName + "'"); + nodeRepository.jobControl().setActive(jobName, active); + return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'"); + } + + private MessageResponse setTargetVersions(HttpRequest request) { + NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase()); + Inspector inspector = toSlime(request.getData()).get(); + List messageParts = new ArrayList<>(3); + + boolean force = inspector.field("force").asBool(); + Inspector versionField = inspector.field("version"); + Inspector osVersionField = inspector.field("osVersion"); + Inspector dockerImageField = inspector.field("dockerImage"); + + if (versionField.valid()) { + Version version = Version.fromString(versionField.asString()); + nodeRepository.infrastructureVersions().setTargetVersion(nodeType, version, force); + messageParts.add("version to " + version.toFullString()); + } + + if (osVersionField.valid()) { + String v = osVersionField.asString(); + if (v.isEmpty()) { + nodeRepository.osVersions().removeTarget(nodeType); + messageParts.add("osVersion to null"); + } else { + Version osVersion = Version.fromString(v); + nodeRepository.osVersions().setTarget(nodeType, osVersion, force); + messageParts.add("osVersion to " + osVersion.toFullString()); + } + } + + if (dockerImageField.valid()) { + Optional dockerImage = Optional.of(dockerImageField.asString()) + .filter(s -> !s.isEmpty()) + .map(DockerImage::fromString); + nodeRepository.dockerImages().setDockerImage(nodeType, dockerImage); + messageParts.add("docker image to " + dockerImage.map(DockerImage::asString).orElse(null)); + } + + if (messageParts.isEmpty()) { + throw new IllegalArgumentException("At least one of 'version', 'osVersion' or 'dockerImage' must be set"); + } + + return new MessageResponse("Set " + String.join(", ", messageParts) + + " for nodes of type " + nodeType); + } + + private MessageResponse cancelFirmwareCheckResponse() { + nodeRepository.firmwareChecks().cancel(); + return new MessageResponse("Cancelled outstanding requests for firmware checks"); + } + + private MessageResponse requestFirmwareCheckResponse() { + nodeRepository.firmwareChecks().request(); + return new MessageResponse("Will request firmware checks on all hosts."); + } + + private static String hostnamesAsString(List nodes) { + return nodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); + } + + private HttpResponse applicationList(URI uri) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor applications = root.setArray("applications"); + for (ApplicationId id : nodeRepository.applications().ids()) { + Cursor application = applications.addObject(); + application.setString("url", withPath("/nodes/v2/applications/" + id.toFullString(), uri).toString()); + application.setString("id", id.toFullString()); + } + return new SlimeJsonResponse(slime); + } + + private HttpResponse application(String id, URI uri) { + Optional application = nodeRepository.applications().get(ApplicationId.fromFullString(id)); + if (application.isEmpty()) + return ErrorResponse.notFoundError("No application '" + id + "'"); + Slime slime = ApplicationSerializer.toSlime(application.get(), + withPath("/nodes/v2/applications/" + id, uri)); + return new SlimeJsonResponse(slime); + } + + /** Returns a copy of the given URI with the host and port from the given URI and the path set to the given path */ + private URI withPath(String newPath, URI uri) { + try { + return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), newPath, null, null); + } + catch (URISyntaxException e) { + throw new RuntimeException("Will not happen", e); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java new file mode 100644 index 00000000000..664aad939f2 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java @@ -0,0 +1,15 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +/** + * Thrown when a resource is not found + * + * @author bratseth + */ +public class NotFoundException extends RuntimeException { + + public NotFoundException(String message) { super(message); } + + public NotFoundException(String message, Throwable cause) { super(message, cause); } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java new file mode 100644 index 00000000000..16858ec6963 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java @@ -0,0 +1,54 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; +import com.yahoo.vespa.hosted.provision.provisioning.DockerImages; +import com.yahoo.vespa.hosted.provision.os.OsVersions; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * A response containing targets for infrastructure Vespa version and OS version. + * + * @author freva + */ +public class UpgradeResponse extends HttpResponse { + + private final InfrastructureVersions infrastructureVersions; + private final OsVersions osVersions; + private final DockerImages dockerImages; + + public UpgradeResponse(InfrastructureVersions infrastructureVersions, OsVersions osVersions, DockerImages dockerImages) { + super(200); + this.infrastructureVersions = infrastructureVersions; + this.osVersions = osVersions; + this.dockerImages = dockerImages; + } + + @Override + public void render(OutputStream stream) throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + Cursor versionsObject = root.setObject("versions"); + infrastructureVersions.getTargetVersions().forEach((nodeType, version) -> versionsObject.setString(nodeType.name(), version.toFullString())); + + Cursor osVersionsObject = root.setObject("osVersions"); + osVersions.targets().forEach((nodeType, osVersion) -> osVersionsObject.setString(nodeType.name(), osVersion.toFullString())); + + + Cursor dockerImagesObject = root.setObject("dockerImages"); + dockerImages.getDockerImages().forEach((nodeType, image) -> dockerImagesObject.setString(nodeType.name(), image.asString())); + + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { return "application/json"; } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ApplicationSerializer.java deleted file mode 100644 index 54f60651795..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ApplicationSerializer.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.provision.applications.Application; -import com.yahoo.vespa.hosted.provision.applications.Cluster; - -import java.net.URI; -import java.util.Collection; - -/** - * Serializes application information for nodes/v2/application responses - */ -public class ApplicationSerializer { - - public static Slime toSlime(Application application, URI applicationUri) { - Slime slime = new Slime(); - toSlime(application, slime.setObject(), applicationUri); - return slime; - } - - private static void toSlime(Application application, Cursor object, URI applicationUri) { - object.setString("url", applicationUri.toString()); - object.setString("id", application.id().toFullString()); - clustersToSlime(application.clusters().values(), object.setObject("clusters")); - } - - private static void clustersToSlime(Collection clusters, Cursor clustersObject) { - clusters.forEach(cluster -> toSlime(cluster, clustersObject.setObject(cluster.id().value()))); - } - - private static void toSlime(Cluster cluster, Cursor clusterObject) { - toSlime(cluster.minResources(), clusterObject.setObject("min")); - toSlime(cluster.maxResources(), clusterObject.setObject("max")); - cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject("suggested"))); - cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject("target"))); - } - - private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { - clusterResourcesObject.setLong("nodes", resources.nodes()); - clusterResourcesObject.setLong("groups", resources.groups()); - NodeResourcesSerializer.toSlime(resources.nodeResources(), clusterResourcesObject.setObject("resources")); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java deleted file mode 100644 index 488b11e627b..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.maintenance.CapacityChecker; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - -/** - * @author mgimle - */ -public class HostCapacityResponse extends HttpResponse { - private final StringBuilder text; - private final Slime slime; - private final CapacityChecker capacityChecker; - private final boolean json; - - public HostCapacityResponse(NodeRepository nodeRepository, HttpRequest request) { - super(200); - capacityChecker = new CapacityChecker(nodeRepository); - - json = request.getBooleanProperty("json"); - String hostsJson = request.getProperty("hosts"); - - text = new StringBuilder(); - slime = new Slime(); - Cursor root = slime.setObject(); - - if (hostsJson != null) { - List hosts = parseHostList(hostsJson); - hostRemovalResponse(root, hosts); - } else { - zoneFailureReponse(root); - } - } - - private List parseHostList(String hosts) { - List hostNames = Arrays.asList(hosts.split(",")); - try { - return capacityChecker.nodesFromHostnames(hostNames); - } catch (IllegalArgumentException e) { - throw new NotFoundException(e.getMessage()); - } - } - - private void hostRemovalResponse(Cursor root, List hosts) { - var failure = capacityChecker.findHostRemovalFailure(hosts); - if (failure.isPresent() && failure.get().failureReason.allocationFailures.size() == 0) { - root.setBool("removalPossible", false); - error(root, "Removing all hosts is trivially impossible."); - } else { - if (json) hostLossPossibleToSlime(root, failure, hosts); - else hostLossPossibleToText(failure, hosts); - } - } - - private void zoneFailureReponse(Cursor root) { - var failurePath = capacityChecker.worstCaseHostLossLeadingToFailure(); - if (failurePath.isPresent()) { - if (json) zoneFailurePathToSlime(root, failurePath.get()); - else zoneFailurePathToText(failurePath.get()); - } else { - error(root, "Node repository contained no hosts."); - } - } - - private void error(Cursor root, String errorMessage) { - if (json) root.setString("error", errorMessage); - else text.append(errorMessage); - } - - private void hostLossPossibleToText(Optional failure, List hostsToRemove) { - text.append(String.format("Attempting to remove %d hosts: ", hostsToRemove.size())); - CapacityChecker.AllocationHistory history = capacityChecker.allocationHistory; - if (failure.isEmpty()) { - text.append("OK\n\n"); - text.append(history); - if (history.oldParents().size() != hostsToRemove.size()) { - long emptyHostCount = hostsToRemove.size() - history.oldParents().size(); - text.append(String.format("\nTrivially removed %d empty host%s.", emptyHostCount, emptyHostCount > 1 ? "s" : "")); - } - } else { - text.append("FAILURE\n\n"); - text.append(history).append("\n"); - text.append(failure.get().failureReason).append("\n\n"); - } - } - - private void zoneFailurePathToText(CapacityChecker.HostFailurePath failurePath) { - text.append(String.format("Found %d hosts. Failure upon trying to remove %d hosts:\n\n", - capacityChecker.getHosts().size(), - failurePath.hostsCausingFailure.size())); - text.append(capacityChecker.allocationHistory).append("\n"); - text.append(failurePath.failureReason); - } - - private void hostLossPossibleToSlime(Cursor root, Optional failure, List hostsToRemove) { - var hosts = root.setArray("hostsToRemove"); - hostsToRemove.forEach(h -> hosts.addString(h.hostname())); - CapacityChecker.AllocationHistory history = capacityChecker.allocationHistory; - root.setBool("removalPossible", failure.isEmpty()); - var arr = root.setArray("history"); - for (var entry : history.historyEntries) { - var object = arr.addObject(); - object.setString("tenant", entry.tenant.hostname()); - if (entry.newParent != null) { - object.setString("newParent", entry.newParent.hostname()); - } - object.setLong("eligibleParents", entry.eligibleParents); - } - } - - private void zoneFailurePathToSlime(Cursor object, CapacityChecker.HostFailurePath failurePath) { - object.setLong("totalHosts", capacityChecker.getHosts().size()); - object.setLong("couldLoseHosts", failurePath.hostsCausingFailure.size()); - failurePath.failureReason.host.ifPresent(host -> - object.setString("failedTenantParent", host.hostname()) - ); - failurePath.failureReason.tenant.ifPresent(tenant -> { - object.setString("failedTenant", tenant.hostname()); - object.setString("failedTenantResources", tenant.flavor().resources().toString()); - tenant.allocation().ifPresent(allocation -> - object.setString("failedTenantAllocation", allocation.toString()) - ); - var explanation = object.setObject("hostCandidateRejectionReasons"); - allocationFailureReasonListToSlime(explanation.setObject("singularReasonFailures"), - failurePath.failureReason.allocationFailures.singularReasonFailures()); - allocationFailureReasonListToSlime(explanation.setObject("totalFailures"), - failurePath.failureReason.allocationFailures); - }); - var details = object.setObject("details"); - hostLossPossibleToSlime(details, Optional.of(failurePath), failurePath.hostsCausingFailure); - } - - private void allocationFailureReasonListToSlime(Cursor root, CapacityChecker.AllocationFailureReasonList allocationFailureReasonList) { - root.setLong("insufficientVcpu", allocationFailureReasonList.insufficientVcpu()); - root.setLong("insufficientMemoryGb", allocationFailureReasonList.insufficientMemoryGb()); - root.setLong("insufficientDiskGb", allocationFailureReasonList.insufficientDiskGb()); - root.setLong("incompatibleDiskSpeed", allocationFailureReasonList.incompatibleDiskSpeed()); - root.setLong("insufficientAvailableIps", allocationFailureReasonList.insufficientAvailableIps()); - root.setLong("violatesParentHostPolicy", allocationFailureReasonList.violatesParentHostPolicy()); - } - - @Override - public void render(OutputStream stream) throws IOException { - if (json) new JsonFormat(true).encode(stream, slime); - else stream.write(text.toString().getBytes()); - } - - @Override - public String getContentType() { - return json ? "application/json" : "text/plain"; - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java deleted file mode 100644 index 4dfdef742d6..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.provision.maintenance.JobControl; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.TreeSet; - -/** A response containing maintenance job status */ -public class JobsResponse extends HttpResponse { - - private final JobControl jobControl; - - public JobsResponse(JobControl jobControl) { - super(200); - this.jobControl = jobControl; - } - - @Override - public void render(OutputStream stream) throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor jobArray = root.setArray("jobs"); - for (String jobName : jobControl.jobs()) - jobArray.addObject().setString("name", jobName); - - Cursor inactiveArray = root.setArray("inactive"); - for (String jobName : new TreeSet<>(jobControl.inactiveJobs())) - inactiveArray.addString(jobName); - - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { return "application/json"; } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java deleted file mode 100644 index 24aa1fe0b25..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.vespa.hosted.provision.NoSuchNodeException; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.yolean.Exceptions; - -import javax.inject.Inject; -import java.util.logging.Level; - -/** - * @author mpolden - */ -public class LoadBalancersApiHandler extends LoggingRequestHandler { - - private final NodeRepository nodeRepository; - - @Inject - public LoadBalancersApiHandler(LoggingRequestHandler.Context parentCtx, NodeRepository nodeRepository) { - super(parentCtx); - this.nodeRepository = nodeRepository; - } - - @Override - public HttpResponse handle(HttpRequest request) { - try { - switch (request.getMethod()) { - case GET: return handleGET(request); - default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - } - } - catch (NotFoundException | NoSuchNodeException e) { - return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); - } - catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); - } - } - - private HttpResponse handleGET(HttpRequest request) { - String path = request.getUri().getPath(); - if (path.equals("/loadbalancers/v1/")) return new LoadBalancersResponse(request, nodeRepository); - throw new NotFoundException("Nothing at path '" + path + "'"); - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java deleted file mode 100644 index 3147d4caded..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; -import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList; -import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.List; -import java.util.Optional; - -/** - * @author mpolden - */ -public class LoadBalancersResponse extends HttpResponse { - - private final NodeRepository nodeRepository; - private final HttpRequest request; - - public LoadBalancersResponse(HttpRequest request, NodeRepository nodeRepository) { - super(200); - this.request = request; - this.nodeRepository = nodeRepository; - } - - private Optional application() { - return Optional.ofNullable(request.getProperty("application")) - .map(ApplicationFilter::toApplicationId); - } - - private List loadBalancers() { - LoadBalancerList loadBalancers; - var application = application(); - if (application.isPresent()) { - loadBalancers = nodeRepository.loadBalancers(application.get()); - } else { - loadBalancers = nodeRepository.loadBalancers(); - } - return loadBalancers.asList(); - } - - @Override - public String getContentType() { return "application/json"; } - - @Override - public void render(OutputStream stream) throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor loadBalancerArray = root.setArray("loadBalancers"); - - loadBalancers().forEach(lb -> { - Cursor lbObject = loadBalancerArray.addObject(); - lbObject.setString("id", lb.id().serializedForm()); - lbObject.setString("state", lb.state().name()); - lbObject.setLong("changedAt", lb.changedAt().toEpochMilli()); - lbObject.setString("application", lb.id().application().application().value()); - lbObject.setString("tenant", lb.id().application().tenant().value()); - lbObject.setString("instance", lb.id().application().instance().value()); - lbObject.setString("cluster", lb.id().cluster().value()); - lbObject.setString("hostname", lb.instance().hostname().value()); - lb.instance().dnsZone().ifPresent(dnsZone -> lbObject.setString("dnsZone", dnsZone.id())); - - Cursor networkArray = lbObject.setArray("networks"); - lb.instance().networks().forEach(networkArray::addString); - - Cursor portArray = lbObject.setArray("ports"); - lb.instance().ports().forEach(portArray::addLong); - - Cursor realArray = lbObject.setArray("reals"); - lb.instance().reals().forEach(real -> { - Cursor realObject = realArray.addObject(); - realObject.setString("hostname", real.hostname().value()); - realObject.setString("ipAddress", real.ipAddress()); - realObject.setLong("port", real.port()); - }); - }); - - new JsonFormat(true).encode(stream, slime); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java deleted file mode 100644 index 9947d2d5108..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.NodeAcl; - -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.util.List; -import java.util.Set; - -/** - * @author mpolden - */ -public class NodeAclResponse extends HttpResponse { - - private static final String CHILDREN_REQUEST_PROPERTY = "children"; - - private final NodeRepository nodeRepository; - private final Slime slime; - private final boolean aclsForChildren; - - public NodeAclResponse(HttpRequest request, NodeRepository nodeRepository) { - super(200); - this.nodeRepository = nodeRepository; - this.slime = new Slime(); - this.aclsForChildren = request.getBooleanProperty(CHILDREN_REQUEST_PROPERTY); - - Cursor root = slime.setObject(); - String hostname = baseName(request.getUri().getPath()); - toSlime(hostname, root); - } - - private void toSlime(String hostname, Cursor object) { - Node node = nodeRepository.getNode(hostname) - .orElseThrow(() -> new NotFoundException("No node with hostname '" + hostname + "'")); - - List acls = nodeRepository.getNodeAcls(node, aclsForChildren); - - Cursor trustedNodesArray = object.setArray("trustedNodes"); - acls.forEach(nodeAcl -> toSlime(nodeAcl, trustedNodesArray)); - - Cursor trustedNetworksArray = object.setArray("trustedNetworks"); - acls.forEach(nodeAcl -> toSlime(nodeAcl.trustedNetworks(), nodeAcl.node(), trustedNetworksArray)); - - Cursor trustedPortsArray = object.setArray("trustedPorts"); - acls.forEach(nodeAcl -> toSlime(nodeAcl.trustedPorts(), nodeAcl, trustedPortsArray)); - } - - private void toSlime(NodeAcl nodeAcl, Cursor array) { - nodeAcl.trustedNodes().forEach(node -> node.ipAddresses().forEach(ipAddress -> { - Cursor object = array.addObject(); - object.setString("hostname", node.hostname()); - object.setString("type", node.type().name()); - object.setString("ipAddress", ipAddress); - object.setString("trustedBy", nodeAcl.node().hostname()); - })); - } - - private void toSlime(Set trustedNetworks, Node trustedby, Cursor array) { - trustedNetworks.forEach(network -> { - Cursor object = array.addObject(); - object.setString("network", network); - object.setString("trustedBy", trustedby.hostname()); - }); - } - - private void toSlime(Set trustedPorts, NodeAcl trustedBy, Cursor array) { - trustedPorts.forEach(port -> { - Cursor object = array.addObject(); - object.setLong("port", port); - object.setString("trustedBy", trustedBy.node().hostname()); - }); - } - - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { - return "application/json"; - } - - private static String baseName(String path) { - return new File(path).getName(); - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java deleted file mode 100644 index 12b245ede7f..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Type; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.provision.LockedNodeList; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.node.Report; -import com.yahoo.vespa.hosted.provision.node.Reports; - -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.time.Clock; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.TreeSet; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; -import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; -import static com.yahoo.config.provision.NodeResources.StorageType.local; -import static com.yahoo.config.provision.NodeResources.StorageType.remote; - -/** - * A class which can take a partial JSON node/v2 node JSON structure and apply it to a node object. - * This is a one-time use object. - * - * @author bratseth - */ -public class NodePatcher { - - private static final String WANT_TO_RETIRE = "wantToRetire"; - - private final NodeFlavors nodeFlavors; - private final Inspector inspector; - private final Supplier nodes; - private final Clock clock; - - private Node node; - - public NodePatcher(NodeFlavors nodeFlavors, InputStream json, Node node, Supplier nodes, Clock clock) { - this.nodeFlavors = nodeFlavors; - this.node = node; - this.nodes = nodes; - this.clock = clock; - try { - this.inspector = SlimeUtils.jsonToSlime(IOUtils.readBytes(json, 1000 * 1000)).get(); - } catch (IOException e) { - throw new UncheckedIOException("Error reading request body", e); - } - } - - /** - * Apply the json to the node and return all nodes affected by the patch. - * More than 1 node may be affected if e.g. the node is a Docker host, which may have - * children that must be updated in a consistent manner. - */ - public List apply() { - List patchedNodes = new ArrayList<>(); - inspector.traverse((String name, Inspector value) -> { - try { - node = applyField(node, name, value); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not set field '" + name + "'", e); - } - - try { - patchedNodes.addAll(applyFieldRecursive(name, value)); - } catch (IllegalArgumentException e) { - // Non recursive field, ignore - } - } ); - patchedNodes.add(node); - - return patchedNodes; - } - - private List applyFieldRecursive(String name, Inspector value) { - switch (name) { - case WANT_TO_RETIRE: - List childNodes = node.type().isDockerHost() ? nodes.get().childrenOf(node).asList() : List.of(); - return childNodes.stream() - .map(child -> applyField(child, name, value)) - .collect(Collectors.toList()); - - default : - throw new IllegalArgumentException("Field " + name + " is not recursive"); - } - } - - private Node applyField(Node node, String name, Inspector value) { - switch (name) { - case "currentRebootGeneration" : - return node.withCurrentRebootGeneration(asLong(value), clock.instant()); - case "currentRestartGeneration" : - return patchCurrentRestartGeneration(asLong(value)); - case "currentDockerImage" : - if (node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) - throw new IllegalArgumentException("Docker image can only be set for docker containers"); - return node.with(node.status().withDockerImage(DockerImage.fromString(asString(value)))); - case "vespaVersion" : - case "currentVespaVersion" : - return node.with(node.status().withVespaVersion(Version.fromString(asString(value)))); - case "currentOsVersion" : - return node.withCurrentOsVersion(Version.fromString(asString(value)), clock.instant()); - case "currentFirmwareCheck": - return node.withFirmwareVerifiedAt(Instant.ofEpochMilli(asLong(value))); - case "failCount" : - return node.with(node.status().withFailCount(asLong(value).intValue())); - case "flavor" : - return node.with(nodeFlavors.getFlavorOrThrow(asString(value))); - case "parentHostname" : - return node.withParentHostname(asString(value)); - case "ipAddresses" : - return IP.Config.verify(node.with(node.ipConfig().with(asStringSet(value))), nodes.get()); - case "additionalIpAddresses" : - return IP.Config.verify(node.with(node.ipConfig().with(IP.Pool.of(asStringSet(value)))), nodes.get()); - case WANT_TO_RETIRE : - return node.withWantToRetire(asBoolean(value), Agent.operator, clock.instant()); - case "wantToDeprovision" : - if (node.type() != NodeType.host && asBoolean(value)) - throw new IllegalArgumentException("wantToDeprovision can only be set for hosts"); - return node.with(node.status().withWantToDeprovision(asBoolean(value))); - case "reports" : - return nodeWithPatchedReports(node, value); - case "openStackId" : - return node.withOpenStackId(asString(value)); - case "diskGb": - case "minDiskAvailableGb": - return node.with(node.flavor().with(node.flavor().resources().withDiskGb(value.asDouble()))); - case "memoryGb": - case "minMainMemoryAvailableGb": - return node.with(node.flavor().with(node.flavor().resources().withMemoryGb(value.asDouble()))); - case "vcpu": - case "minCpuCores": - return node.with(node.flavor().with(node.flavor().resources().withVcpu(value.asDouble()))); - case "fastDisk": - return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? fast : slow))); - case "remoteStorage": - return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? remote : local))); - case "bandwidthGbps": - return node.with(node.flavor().with(node.flavor().resources().withBandwidthGbps(value.asDouble()))); - case "modelName": - return value.type() == Type.NIX ? node.withoutModelName() : node.withModelName(asString(value)); - case "requiredDiskSpeed": - return patchRequiredDiskSpeed(asString(value)); - case "reservedTo": - return value.type() == Type.NIX ? node.withoutReservedTo() : node.withReservedTo(TenantName.from(value.asString())); - default : - throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); - } - } - - private Node nodeWithPatchedReports(Node node, Inspector reportsInspector) { - Node patchedNode; - // "reports": null clears the reports - if (reportsInspector.type() == Type.NIX) { - patchedNode = node.with(new Reports()); - } else { - var reportsBuilder = new Reports.Builder(node.reports()); - reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> { - if (reportInspector.type() == Type.NIX) { - // ... "reports": { "reportId": null } clears the report "reportId" - reportsBuilder.clearReport(reportId); - } else { - // ... "reports": { "reportId": {...} } overrides the whole report "reportId" - reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector)); - } - }); - patchedNode = node.with(reportsBuilder.build()); - } - - boolean hadHardFailReports = node.reports().getReports().stream() - .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL); - boolean hasHardFailReports = patchedNode.reports().getReports().stream() - .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL); - - // If this patch resulted in going from not having HARD_FAIL report to having one, or vice versa - if (hadHardFailReports != hasHardFailReports) { - // Do not automatically change wantToDeprovision when - // 1. Transitioning to having a HARD_FAIL report and being in state failed: - // To allow operators manually unset before the host is parked and deleted. - // 2. When in parked state: Deletion is imminent, possibly already underway - if ((hasHardFailReports && node.state() == Node.State.failed) || node.state() == Node.State.parked) - return patchedNode; - - patchedNode = patchedNode.with(patchedNode.status().withWantToDeprovision(hasHardFailReports)); - } - - return patchedNode; - } - - private Set asStringSet(Inspector field) { - if ( ! field.type().equals(Type.ARRAY)) - throw new IllegalArgumentException("Expected an ARRAY value, got a " + field.type()); - - TreeSet strings = new TreeSet<>(); - for (int i = 0; i < field.entries(); i++) { - Inspector entry = field.entry(i); - if ( ! entry.type().equals(Type.STRING)) - throw new IllegalArgumentException("Expected a STRING value, got a " + entry.type()); - strings.add(entry.asString()); - } - - return strings; - } - - private Node patchRequiredDiskSpeed(String value) { - Optional allocation = node.allocation(); - if (allocation.isPresent()) - return node.with(allocation.get().withRequestedResources( - allocation.get().requestedResources().with(NodeResources.DiskSpeed.valueOf(value)))); - else - throw new IllegalArgumentException("Node is not allocated"); - } - - private Node patchCurrentRestartGeneration(Long value) { - Optional allocation = node.allocation(); - if (allocation.isPresent()) - return node.with(allocation.get().withRestart(allocation.get().restartGeneration().withCurrent(value))); - else - throw new IllegalArgumentException("Node is not allocated"); - } - - private Long asLong(Inspector field) { - if ( ! field.type().equals(Type.LONG)) - throw new IllegalArgumentException("Expected a LONG value, got a " + field.type()); - return field.asLong(); - } - - private String asString(Inspector field) { - if ( ! field.type().equals(Type.STRING)) - throw new IllegalArgumentException("Expected a STRING value, got a " + field.type()); - return field.asString(); - } - - private Optional asOptionalString(Inspector field) { - return field.type().equals(Type.NIX) ? Optional.empty() : Optional.of(asString(field)); - } - - // Allows us to clear optional flags by passing "null" as slime does not have an empty (but present) representation - private Optional removeQuotedNulls(Optional value) { - return value.filter(v -> !v.equals("null")); - } - - private boolean asBoolean(Inspector field) { - if ( ! field.type().equals(Type.BOOL)) - throw new IllegalArgumentException("Expected a BOOL value, got a " + field.type()); - return field.asBool(); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java deleted file mode 100644 index 0d0c9305a92..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeResourcesSerializer.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.config.provision.NodeResources; -import com.yahoo.slime.Cursor; - -/** - * @author bratseth - */ -public class NodeResourcesSerializer { - - static void toSlime(NodeResources resources, Cursor object) { - object.setDouble("vcpu", resources.vcpu()); - object.setDouble("memoryGb", resources.memoryGb()); - object.setDouble("diskGb", resources.diskGb()); - object.setDouble("bandwidthGbps", resources.bandwidthGbps()); - object.setString("diskSpeed", toString(resources.diskSpeed())); - object.setString("storageType", toString(resources.storageType())); - } - - public static NodeResources.DiskSpeed diskSpeedFrom(String diskSpeed) { - switch (diskSpeed) { - case "fast": return NodeResources.DiskSpeed.fast; - case "slow": return NodeResources.DiskSpeed.slow; - case "any" : return NodeResources.DiskSpeed.any; - default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed + "'"); - } - } - - private static String toString(NodeResources.DiskSpeed diskSpeed) { - switch (diskSpeed) { - case fast : return "fast"; - case slow : return "slow"; - case any : return "any"; - default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed.name() + "'"); - } - } - - public static NodeResources.StorageType storageTypeFrom(String storageType) { - switch (storageType) { - case "local" : return NodeResources.StorageType.local; - case "remote": return NodeResources.StorageType.remote; - case "any" : return NodeResources.StorageType.any; - default: throw new IllegalArgumentException("Unknown storage type '" + storageType + "'"); - } - } - - private static String toString(NodeResources.StorageType storageType) { - switch (storageType) { - case remote : return "remote"; - case local : return "local"; - case any : return "any"; - default: throw new IllegalArgumentException("Unknown storage type '" + storageType.name() + "'"); - } - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java deleted file mode 100644 index 67f3b293de8..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.slime.Cursor; -import com.yahoo.vespa.hosted.provision.Node; - -/** - * Serializer for node wire format. Used to serialize node objects sent over the wire (HTTP). - * - * @author mpolden - */ -public class NodeSerializer { - - public static Node.State stateFrom(String state) { - switch (state) { - case "active": return Node.State.active; - case "dirty": return Node.State.dirty; - case "failed": return Node.State.failed; - case "inactive": return Node.State.inactive; - case "parked": return Node.State.parked; - case "provisioned": return Node.State.provisioned; - case "ready": return Node.State.ready; - case "reserved": return Node.State.reserved; - case "deprovisioned": return Node.State.deprovisioned; - default: throw new IllegalArgumentException("Unknown node state '" + state + "'"); - } - } - - public static String toString(Node.State state) { - switch (state) { - case active: return "active"; - case dirty: return "dirty"; - case failed: return "failed"; - case inactive: return "inactive"; - case parked: return "parked"; - case provisioned: return "provisioned"; - case ready: return "ready"; - case reserved: return "reserved"; - case deprovisioned: return "deprovisioned"; - default: throw new IllegalArgumentException("Unknown node state '" + state + "'"); - } - } - - public static NodeType typeFrom(String nodeType) { - switch (nodeType) { - case "tenant": return NodeType.tenant; - case "host": return NodeType.host; - case "proxy": return NodeType.proxy; - case "proxyhost": return NodeType.proxyhost; - case "config": return NodeType.config; - case "confighost": return NodeType.confighost; - case "controller": return NodeType.controller; - case "controllerhost": return NodeType.controllerhost; - case "devhost": return NodeType.devhost; - default: throw new IllegalArgumentException("Unknown node type '" + nodeType + "'"); - } - } - - public static String toString(NodeType type) { - switch (type) { - case tenant: return "tenant"; - case host: return "host"; - case proxy: return "proxy"; - case proxyhost: return "proxyhost"; - case config: return "config"; - case confighost: return "confighost"; - case controller: return "controller"; - case controllerhost: return "controllerhost"; - case devhost: return "devhost"; - default: throw new IllegalArgumentException("Unknown node type '" + type.name() + "'"); - } - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java deleted file mode 100644 index 6799b68daae..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ /dev/null @@ -1,436 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.HostFilter; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.io.IOUtils; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.ResourceResponse; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.slime.Type; -import com.yahoo.vespa.hosted.provision.NoSuchNodeException; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.applications.Application; -import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; -import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; -import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; -import com.yahoo.vespa.hosted.provision.node.filter.NodeOsVersionFilter; -import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; -import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; -import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; -import com.yahoo.vespa.hosted.provision.restapi.v2.NodesResponse.ResponseType; -import com.yahoo.vespa.orchestrator.Orchestrator; -import com.yahoo.yolean.Exceptions; - -import javax.inject.Inject; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.logging.Level; -import java.util.stream.Collectors; - -import static com.yahoo.slime.SlimeUtils.optionalString; - -/** - * The implementation of the /nodes/v2 API. - * See RestApiTest for documentation. - * - * @author bratseth - */ -public class NodesApiHandler extends LoggingRequestHandler { - - private final Orchestrator orchestrator; - private final NodeRepository nodeRepository; - private final NodeFlavors nodeFlavors; - - @Inject - public NodesApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator, - NodeRepository nodeRepository, NodeFlavors flavors) { - super(parentCtx); - this.orchestrator = orchestrator; - this.nodeRepository = nodeRepository; - this.nodeFlavors = flavors; - } - - @Override - public HttpResponse handle(HttpRequest request) { - try { - switch (request.getMethod()) { - case GET: return handleGET(request); - case PUT: return handlePUT(request); - case POST: return isPatchOverride(request) ? handlePATCH(request) : handlePOST(request); - case DELETE: return handleDELETE(request); - case PATCH: return handlePATCH(request); - default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - } - } - catch (NotFoundException | NoSuchNodeException e) { - return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); - } - catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); - } - } - - private HttpResponse handleGET(HttpRequest request) { - Path path = new Path(request.getUri()); - String pathS = request.getUri().getPath(); - if (pathS.equals( "/nodes/v2/")) return new ResourceResponse(request.getUri(), "state", "node", "command", "maintenance", "upgrade", "application"); - if (pathS.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository); - if (pathS.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository); - if (pathS.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, orchestrator, nodeRepository); - if (pathS.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, orchestrator, nodeRepository); - if (pathS.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); - if (pathS.equals( "/nodes/v2/command/")) return new ResourceResponse(request.getUri(), "restart", "reboot"); - if (pathS.equals( "/nodes/v2/maintenance/")) return new JobsResponse(nodeRepository.jobControl()); - if (pathS.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(nodeRepository.infrastructureVersions(), nodeRepository.osVersions(), nodeRepository.dockerImages()); - if (pathS.startsWith("/nodes/v2/capacity")) return new HostCapacityResponse(nodeRepository, request); - if (path.matches("/nodes/v2/application")) return applicationList(request.getUri()); - if (path.matches("/nodes/v2/application/{applicationId}")) return application(path.get("applicationId"), request.getUri()); - throw new NotFoundException("Nothing at " + path); - } - - private HttpResponse handlePUT(HttpRequest request) { - String path = request.getUri().getPath(); - // Check paths to disallow illegal state changes - if (path.startsWith("/nodes/v2/state/ready/") || - path.startsWith("/nodes/v2/state/availablefornewallocations/")) { - nodeRepository.markNodeAvailableForNewAllocation(lastElement(path), Agent.operator, "Readied through the nodes/v2 API"); - return new MessageResponse("Moved " + lastElement(path) + " to ready"); - } - else if (path.startsWith("/nodes/v2/state/failed/")) { - List failedNodes = nodeRepository.failRecursively(lastElement(path), Agent.operator, "Failed through the nodes/v2 API"); - return new MessageResponse("Moved " + hostnamesAsString(failedNodes) + " to failed"); - } - else if (path.startsWith("/nodes/v2/state/parked/")) { - List parkedNodes = nodeRepository.parkRecursively(lastElement(path), Agent.operator, "Parked through the nodes/v2 API"); - return new MessageResponse("Moved " + hostnamesAsString(parkedNodes) + " to parked"); - } - else if (path.startsWith("/nodes/v2/state/dirty/")) { - List dirtiedNodes = nodeRepository.dirtyRecursively(lastElement(path), Agent.operator, "Dirtied through the nodes/v2 API"); - return new MessageResponse("Moved " + hostnamesAsString(dirtiedNodes) + " to dirty"); - } - else if (path.startsWith("/nodes/v2/state/active/")) { - nodeRepository.reactivate(lastElement(path), Agent.operator, "Reactivated through nodes/v2 API"); - return new MessageResponse("Moved " + lastElement(path) + " to active"); - } - - throw new NotFoundException("Cannot put to path '" + path + "'"); - } - - private HttpResponse handlePATCH(HttpRequest request) { - String path = request.getUri().getPath(); - if (path.startsWith("/nodes/v2/node/")) { - // TODO: Node is fetched outside of lock, might change after getting lock - Node node = nodeFromRequest(request); - try (var lock = nodeRepository.lock(node)) { - var patchedNodes = new NodePatcher(nodeFlavors, request.getData(), node, () -> nodeRepository.list(lock), - nodeRepository.clock()).apply(); - nodeRepository.write(patchedNodes, lock); - } - return new MessageResponse("Updated " + node.hostname()); - } - else if (path.startsWith("/nodes/v2/upgrade/")) { - return setTargetVersions(request); - } - - throw new NotFoundException("Nothing at '" + path + "'"); - } - - private HttpResponse handlePOST(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/nodes/v2/command/restart")) { - int restartCount = nodeRepository.restart(toNodeFilter(request)).size(); - return new MessageResponse("Scheduled restart of " + restartCount + " matching nodes"); - } - if (path.matches("/nodes/v2/command/reboot")) { - int rebootCount = nodeRepository.reboot(toNodeFilter(request)).size(); - return new MessageResponse("Scheduled reboot of " + rebootCount + " matching nodes"); - } - if (path.matches("/nodes/v2/node")) { - int addedNodes = addNodes(request.getData()); - return new MessageResponse("Added " + addedNodes + " nodes to the provisioned state"); - } - if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), false); - if (path.matches("/nodes/v2/maintenance/run/{job}")) return runJob(path.get("job")); - if (path.matches("/nodes/v2/upgrade/firmware")) return requestFirmwareCheckResponse(); - - throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); - } - - private HttpResponse runJob(String job) { - nodeRepository.jobControl().run(job); - return new MessageResponse("Executed job '" + job + "'"); - } - - private HttpResponse handleDELETE(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/nodes/v2/node/{hostname}")) { - String hostname = path.get("hostname"); - List removedNodes = nodeRepository.removeRecursively(hostname); - return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", "))); - } - if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), true); - if (path.matches("/nodes/v2/upgrade/firmware")) return cancelFirmwareCheckResponse(); - - throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); - } - - private Node nodeFromRequest(HttpRequest request) { - String hostname = lastElement(request.getUri().getPath()); - return nodeRepository.getNode(hostname).orElseThrow(() -> - new NotFoundException("No node found with hostname " + hostname)); - } - - public int addNodes(InputStream jsonStream) { - List nodes = createNodesFromSlime(toSlime(jsonStream).get()); - return nodeRepository.addNodes(nodes, Agent.operator).size(); - } - - private Slime toSlime(InputStream jsonStream) { - try { - byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000); - return SlimeUtils.jsonToSlime(jsonBytes); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private List createNodesFromSlime(Inspector object) { - List nodes = new ArrayList<>(); - object.traverse((ArrayTraverser) (int i, Inspector item) -> nodes.add(createNode(item))); - return nodes; - } - - private Node createNode(Inspector inspector) { - Optional parentHostname = optionalString(inspector.field("parentHostname")); - Optional modelName = optionalString(inspector.field("modelName")); - Set ipAddresses = new HashSet<>(); - inspector.field("ipAddresses").traverse((ArrayTraverser) (i, item) -> ipAddresses.add(item.asString())); - Set ipAddressPool = new HashSet<>(); - inspector.field("additionalIpAddresses").traverse((ArrayTraverser) (i, item) -> ipAddressPool.add(item.asString())); - - return Node.create(inspector.field("openStackId").asString(), - new IP.Config(ipAddresses, ipAddressPool), - inspector.field("hostname").asString(), - parentHostname, - modelName, - flavorFromSlime(inspector), - reservedToFromSlime(inspector.field("reservedTo")), - nodeTypeFromSlime(inspector.field("type")) - ); - } - - private Flavor flavorFromSlime(Inspector inspector) { - Inspector flavorInspector = inspector.field("flavor"); - Inspector resourcesInspector = inspector.field("resources"); - if ( ! flavorInspector.valid()) { - return new Flavor(new NodeResources( - requiredField(resourcesInspector, "vcpu", Inspector::asDouble), - requiredField(resourcesInspector, "memoryGb", Inspector::asDouble), - requiredField(resourcesInspector, "diskGb", Inspector::asDouble), - requiredField(resourcesInspector, "bandwidthGbps", Inspector::asDouble), - optionalString(resourcesInspector.field("diskSpeed")).map(NodeResourcesSerializer::diskSpeedFrom).orElse(NodeResources.DiskSpeed.getDefault()), - optionalString(resourcesInspector.field("storageType")).map(NodeResourcesSerializer::storageTypeFrom).orElse(NodeResources.StorageType.getDefault()))); - } - - Flavor flavor = nodeFlavors.getFlavorOrThrow(flavorInspector.asString()); - if (resourcesInspector.valid()) { - if (resourcesInspector.field("vcpu").valid()) - flavor = flavor.with(flavor.resources().withVcpu(resourcesInspector.field("vcpu").asDouble())); - if (resourcesInspector.field("memoryGb").valid()) - flavor = flavor.with(flavor.resources().withMemoryGb(resourcesInspector.field("memoryGb").asDouble())); - if (resourcesInspector.field("diskGb").valid()) - flavor = flavor.with(flavor.resources().withDiskGb(resourcesInspector.field("diskGb").asDouble())); - if (resourcesInspector.field("bandwidthGbps").valid()) - flavor = flavor.with(flavor.resources().withBandwidthGbps(resourcesInspector.field("bandwidthGbps").asDouble())); - if (resourcesInspector.field("diskSpeed").valid()) - flavor = flavor.with(flavor.resources().with(NodeResourcesSerializer.diskSpeedFrom(resourcesInspector.field("diskSpeed").asString()))); - if (resourcesInspector.field("storageType").valid()) - flavor = flavor.with(flavor.resources().with(NodeResourcesSerializer.storageTypeFrom(resourcesInspector.field("storageType").asString()))); - } - return flavor; - } - - private static T requiredField(Inspector inspector, String fieldName, Function valueExtractor) { - Inspector field = inspector.field(fieldName); - if (!field.valid()) throw new IllegalArgumentException("Required field '" + fieldName + "' is missing"); - return valueExtractor.apply(field); - } - - private NodeType nodeTypeFromSlime(Inspector object) { - if (! object.valid()) return NodeType.tenant; // default - return NodeSerializer.typeFrom(object.asString()); - } - - private Optional reservedToFromSlime(Inspector object) { - if (! object.valid()) return Optional.empty(); - if ( object.type() != Type.STRING) - throw new IllegalArgumentException("Expected 'reservedTo' to be a string but is " + object); - return Optional.of(TenantName.from(object.asString())); - } - - public static NodeFilter toNodeFilter(HttpRequest request) { - NodeFilter filter = NodeHostFilter.from(HostFilter.from(request.getProperty("hostname"), - request.getProperty("flavor"), - request.getProperty("clusterType"), - request.getProperty("clusterId"))); - filter = ApplicationFilter.from(request.getProperty("application"), filter); - filter = StateFilter.from(request.getProperty("state"), filter); - filter = NodeTypeFilter.from(request.getProperty("type"), filter); - filter = ParentHostFilter.from(request.getProperty("parentHost"), filter); - filter = NodeOsVersionFilter.from(request.getProperty("osVersion"), filter); - return filter; - } - - private static String lastElement(String path) { - if (path.endsWith("/")) - path = path.substring(0, path.length()-1); - int lastSlash = path.lastIndexOf("/"); - if (lastSlash < 0) return path; - return path.substring(lastSlash + 1); - } - - private static boolean isPatchOverride(HttpRequest request) { - // Since Jersey's HttpUrlConnector does not support PATCH we support this by override this on POST requests. - String override = request.getHeader("X-HTTP-Method-Override"); - if (override != null) { - if (override.equals("PATCH")) { - return true; - } else { - String msg = String.format("Illegal X-HTTP-Method-Override header for POST request. Accepts 'PATCH' but got '%s'", override); - throw new IllegalArgumentException(msg); - } - } - return false; - } - - private MessageResponse setJobActive(String jobName, boolean active) { - if ( ! nodeRepository.jobControl().jobs().contains(jobName)) - throw new NotFoundException("No job named '" + jobName + "'"); - nodeRepository.jobControl().setActive(jobName, active); - return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'"); - } - - private MessageResponse setTargetVersions(HttpRequest request) { - NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase()); - Inspector inspector = toSlime(request.getData()).get(); - List messageParts = new ArrayList<>(3); - - boolean force = inspector.field("force").asBool(); - Inspector versionField = inspector.field("version"); - Inspector osVersionField = inspector.field("osVersion"); - Inspector dockerImageField = inspector.field("dockerImage"); - - if (versionField.valid()) { - Version version = Version.fromString(versionField.asString()); - nodeRepository.infrastructureVersions().setTargetVersion(nodeType, version, force); - messageParts.add("version to " + version.toFullString()); - } - - if (osVersionField.valid()) { - String v = osVersionField.asString(); - if (v.isEmpty()) { - nodeRepository.osVersions().removeTarget(nodeType); - messageParts.add("osVersion to null"); - } else { - Version osVersion = Version.fromString(v); - nodeRepository.osVersions().setTarget(nodeType, osVersion, force); - messageParts.add("osVersion to " + osVersion.toFullString()); - } - } - - if (dockerImageField.valid()) { - Optional dockerImage = Optional.of(dockerImageField.asString()) - .filter(s -> !s.isEmpty()) - .map(DockerImage::fromString); - nodeRepository.dockerImages().setDockerImage(nodeType, dockerImage); - messageParts.add("docker image to " + dockerImage.map(DockerImage::asString).orElse(null)); - } - - if (messageParts.isEmpty()) { - throw new IllegalArgumentException("At least one of 'version', 'osVersion' or 'dockerImage' must be set"); - } - - return new MessageResponse("Set " + String.join(", ", messageParts) + - " for nodes of type " + nodeType); - } - - private MessageResponse cancelFirmwareCheckResponse() { - nodeRepository.firmwareChecks().cancel(); - return new MessageResponse("Cancelled outstanding requests for firmware checks"); - } - - private MessageResponse requestFirmwareCheckResponse() { - nodeRepository.firmwareChecks().request(); - return new MessageResponse("Will request firmware checks on all hosts."); - } - - private static String hostnamesAsString(List nodes) { - return nodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); - } - - private HttpResponse applicationList(URI uri) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor applications = root.setArray("applications"); - for (ApplicationId id : nodeRepository.applications().ids()) { - Cursor application = applications.addObject(); - application.setString("url", withPath("/nodes/v2/applications/" + id.toFullString(), uri).toString()); - application.setString("id", id.toFullString()); - } - return new SlimeJsonResponse(slime); - } - - private HttpResponse application(String id, URI uri) { - Optional application = nodeRepository.applications().get(ApplicationId.fromFullString(id)); - if (application.isEmpty()) - return ErrorResponse.notFoundError("No application '" + id + "'"); - Slime slime = ApplicationSerializer.toSlime(application.get(), - withPath("/nodes/v2/applications/" + id, uri)); - return new SlimeJsonResponse(slime); - } - - /** Returns a copy of the given URI with the host and port from the given URI and the path set to the given path */ - private URI withPath(String newPath, URI uri) { - try { - return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), newPath, null, null); - } - catch (URISyntaxException e) { - throw new RuntimeException("Will not happen", e); - } - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java deleted file mode 100644 index c6b92d0a7f5..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.serialization.NetworkPortsSerializer; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.History; -import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; -import com.yahoo.vespa.orchestrator.Orchestrator; -import com.yahoo.vespa.orchestrator.status.HostInfo; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; - -/** -* @author bratseth -*/ -class NodesResponse extends HttpResponse { - - /** The responses this can create */ - public enum ResponseType { nodeList, stateList, nodesInStateList, singleNode } - - /** The request url minus parameters, with a trailing slash added if missing */ - private final String parentUrl; - - /** The parent url of nodes */ - private final String nodeParentUrl; - - private final NodeFilter filter; - private final boolean recursive; - private final Function> orchestrator; - private final NodeRepository nodeRepository; - private final Slime slime; - - public NodesResponse(ResponseType responseType, HttpRequest request, - Orchestrator orchestrator, NodeRepository nodeRepository) { - super(200); - this.parentUrl = toParentUrl(request); - this.nodeParentUrl = toNodeParentUrl(request); - filter = NodesApiHandler.toNodeFilter(request); - this.recursive = request.getBooleanProperty("recursive"); - this.orchestrator = orchestrator.getHostResolver(); - this.nodeRepository = nodeRepository; - - slime = new Slime(); - Cursor root = slime.setObject(); - switch (responseType) { - case nodeList: nodesToSlime(root); break; - case stateList : statesToSlime(root); break; - case nodesInStateList: nodesToSlime(NodeSerializer.stateFrom(lastElement(parentUrl)), root); break; - case singleNode : nodeToSlime(lastElement(parentUrl), root); break; - default: throw new IllegalArgumentException(); - } - } - - private String toParentUrl(HttpRequest request) { - URI uri = request.getUri(); - String parentUrl = uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + uri.getPath(); - if ( ! parentUrl.endsWith("/")) - parentUrl = parentUrl + "/"; - return parentUrl; - } - - private String toNodeParentUrl(HttpRequest request) { - URI uri = request.getUri(); - return uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + "/nodes/v2/node/"; - } - - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { - return "application/json"; - } - - private void statesToSlime(Cursor root) { - Cursor states = root.setObject("states"); - for (Node.State state : Node.State.values()) - toSlime(state, states.setObject(NodeSerializer.toString(state))); - } - - private void toSlime(Node.State state, Cursor object) { - object.setString("url", parentUrl + NodeSerializer.toString(state)); - if (recursive) - nodesToSlime(state, object); - } - - /** Outputs the nodes in the given state to a node array */ - private void nodesToSlime(Node.State state, Cursor parentObject) { - Cursor nodeArray = parentObject.setArray("nodes"); - for (NodeType type : NodeType.values()) - toSlime(nodeRepository.getNodes(type, state), nodeArray); - } - - /** Outputs all the nodes to a node array */ - private void nodesToSlime(Cursor parentObject) { - Cursor nodeArray = parentObject.setArray("nodes"); - toSlime(nodeRepository.getNodes(), nodeArray); - } - - private void toSlime(List nodes, Cursor array) { - for (Node node : nodes) { - if ( ! filter.matches(node)) continue; - toSlime(node, recursive, array.addObject()); - } - } - - private void nodeToSlime(String hostname, Cursor object) { - Node node = nodeRepository.getNode(hostname).orElseThrow(() -> - new NotFoundException("No node with hostname '" + hostname + "'")); - toSlime(node, true, object); - } - - private void toSlime(Node node, boolean allFields, Cursor object) { - object.setString("url", nodeParentUrl + node.hostname()); - if ( ! allFields) return; - object.setString("id", node.hostname()); - object.setString("state", NodeSerializer.toString(node.state())); - object.setString("type", node.type().name()); - object.setString("hostname", node.hostname()); - object.setString("type", NodeSerializer.toString(node.type())); - if (node.parentHostname().isPresent()) { - object.setString("parentHostname", node.parentHostname().get()); - } - object.setString("openStackId", node.id()); - object.setString("flavor", node.flavor().name()); - node.reservedTo().ifPresent(reservedTo -> object.setString("reservedTo", reservedTo.value())); - if (node.flavor().isConfigured()) - object.setDouble("cpuCores", node.flavor().getMinCpuCores()); - NodeResourcesSerializer.toSlime(node.flavor().resources(), object.setObject("resources")); - if (node.flavor().cost() > 0) - object.setLong("cost", node.flavor().cost()); - object.setString("environment", node.flavor().getType().name()); - node.allocation().ifPresent(allocation -> { - toSlime(allocation.owner(), object.setObject("owner")); - toSlime(allocation.membership(), object.setObject("membership")); - object.setLong("restartGeneration", allocation.restartGeneration().wanted()); - object.setLong("currentRestartGeneration", allocation.restartGeneration().current()); - object.setString("wantedDockerImage", allocation.membership().cluster().dockerImage() - .orElseGet(() -> nodeRepository.dockerImage(node).withTag(allocation.membership().cluster().vespaVersion()).asString())); - object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); - NodeResourcesSerializer.toSlime(allocation.requestedResources(), object.setObject("requestedResources")); - allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); - orchestrator.apply(new HostName(node.hostname())) - .ifPresent(info -> { - object.setBool("allowedToBeDown", info.status().isSuspended()); - info.suspendedSince().ifPresent(since -> object.setLong("suspendedSinceMillis", since.toEpochMilli())); - }); - }); - object.setLong("rebootGeneration", node.status().reboot().wanted()); - object.setLong("currentRebootGeneration", node.status().reboot().current()); - node.status().osVersion().current().ifPresent(version -> object.setString("currentOsVersion", version.toFullString())); - node.status().osVersion().wanted().ifPresent(version -> object.setString("wantedOsVersion", version.toFullString())); - node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong("currentFirmwareCheck", instant.toEpochMilli())); - if (node.type().isDockerHost()) - nodeRepository.firmwareChecks().requiredAfter().ifPresent(after -> object.setLong("wantedFirmwareCheck", after.toEpochMilli())); - node.status().vespaVersion().ifPresent(version -> object.setString("vespaVersion", version.toFullString())); - currentDockerImage(node).ifPresent(dockerImage -> object.setString("currentDockerImage", dockerImage.asString())); - object.setLong("failCount", node.status().failCount()); - object.setBool("wantToRetire", node.status().wantToRetire()); - object.setBool("wantToDeprovision", node.status().wantToDeprovision()); - toSlime(node.history(), object.setArray("history")); - ipAddressesToSlime(node.ipAddresses(), object.setArray("ipAddresses")); - ipAddressesToSlime(node.ipAddressPool().asSet(), object.setArray("additionalIpAddresses")); - node.reports().toSlime(object, "reports"); - node.modelName().ifPresent(modelName -> object.setString("modelName", modelName)); - } - - private void toSlime(ApplicationId id, Cursor object) { - object.setString("tenant", id.tenant().value()); - object.setString("application", id.application().value()); - object.setString("instance", id.instance().value()); - } - - private void toSlime(ClusterMembership membership, Cursor object) { - object.setString("clustertype", membership.cluster().type().name()); - object.setString("clusterid", membership.cluster().id().value()); - object.setString("group", String.valueOf(membership.cluster().group().get().index())); - object.setLong("index", membership.index()); - object.setBool("retired", membership.retired()); - } - - private void toSlime(History history, Cursor array) { - for (History.Event event : history.events()) { - Cursor object = array.addObject(); - object.setString("event", event.type().name()); - object.setLong("at", event.at().toEpochMilli()); - object.setString("agent", event.agent().name()); - } - } - - // Hack: For non-docker nodes, return current docker image as default prefix + current Vespa version - // TODO: Remove current + wanted docker image from response for non-docker types - private Optional currentDockerImage(Node node) { - return node.status().dockerImage() - .or(() -> Optional.of(node) - .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) - .flatMap(n -> n.status().vespaVersion() - .map(version -> nodeRepository.dockerImage(n).withTag(version)))); - } - - private void ipAddressesToSlime(Set ipAddresses, Cursor array) { - ipAddresses.forEach(array::addString); - } - - private String lastElement(String path) { - if (path.endsWith("/")) - path = path.substring(0, path.length()-1); - int lastSlash = path.lastIndexOf("/"); - if (lastSlash < 0) return path; - return path.substring(lastSlash+1); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java deleted file mode 100644 index a2bfba95b8a..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -/** - * Thrown when a resource is not found - * - * @author bratseth - */ -public class NotFoundException extends RuntimeException { - - public NotFoundException(String message) { super(message); } - - public NotFoundException(String message, Throwable cause) { super(message, cause); } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java deleted file mode 100644 index 381a1bc27aa..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; -import com.yahoo.vespa.hosted.provision.provisioning.DockerImages; -import com.yahoo.vespa.hosted.provision.os.OsVersions; - -import java.io.IOException; -import java.io.OutputStream; - -/** - * A response containing targets for infrastructure Vespa version and OS version. - * - * @author freva - */ -public class UpgradeResponse extends HttpResponse { - - private final InfrastructureVersions infrastructureVersions; - private final OsVersions osVersions; - private final DockerImages dockerImages; - - public UpgradeResponse(InfrastructureVersions infrastructureVersions, OsVersions osVersions, DockerImages dockerImages) { - super(200); - this.infrastructureVersions = infrastructureVersions; - this.osVersions = osVersions; - this.dockerImages = dockerImages; - } - - @Override - public void render(OutputStream stream) throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - Cursor versionsObject = root.setObject("versions"); - infrastructureVersions.getTargetVersions().forEach((nodeType, version) -> versionsObject.setString(nodeType.name(), version.toFullString())); - - Cursor osVersionsObject = root.setObject("osVersions"); - osVersions.targets().forEach((nodeType, osVersion) -> osVersionsObject.setString(nodeType.name(), osVersion.toFullString())); - - - Cursor dockerImagesObject = root.setObject("dockerImages"); - dockerImages.getDockerImages().forEach((nodeType, image) -> dockerImagesObject.setString(nodeType.name(), image.asString())); - - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { return "application/json"; } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index d26accd7a84..d66bb7d6efb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -32,10 +32,10 @@ public class ContainerConfig { " \n" + " \n" + " \n" + - " \n" + + " \n" + " http://*/nodes/v2/*\n" + " \n" + - " \n" + + " \n" + " http://*/loadbalancers/v1/*\n" + " \n" + " \n" + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java new file mode 100644 index 00000000000..e9811985b7d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.application.container.handler.Request; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class LoadBalancersV1ApiTest { + + private RestApiTester tester; + + @Before + public void createTester() { + tester = new RestApiTester(); + } + + @After + public void closeTester() { + tester.close(); + } + + @Test + public void test_load_balancers() throws Exception { + tester.assertFile(new Request("http://localhost:8080/loadbalancers/v1/"), "load-balancers.json"); + tester.assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant4.application4.instance4"), "load-balancers-single.json"); + tester.assertResponse(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant.nonexistent.default"), "{\"loadBalancers\":[]}"); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializerTest.java new file mode 100644 index 00000000000..9bc27c0cc04 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializerTest.java @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + + +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.restapi.NodeSerializer; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class NodeSerializerTest { + + @Test + public void serialize_node_types() { + for (NodeType t : NodeType.values()) { + assertEquals(t, NodeSerializer.typeFrom(NodeSerializer.toString(t))); + } + } + + @Test + public void serialize_node_states() { + for (Node.State s : Node.State.values()) { + assertEquals(s, NodeSerializer.stateFrom(NodeSerializer.toString(s))); + } + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java new file mode 100644 index 00000000000..3e23b424675 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -0,0 +1,984 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.JDisc; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; +import com.yahoo.io.IOUtils; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.maintenance.OsUpgradeActivator; +import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; +import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; +import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; +import org.junit.After; +import org.junit.Before; +import org.junit.ComparisonFailure; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * Test of the REST APIs provided by the node repository. + * + * Note: This class is referenced from our operations documentation and must not be renamed/moved without updating that. + * + * @author bratseth + */ +public class NodesV2ApiTest { + + private RestApiTester tester; + + @Before + public void createTester() { + tester = new RestApiTester(); + } + + @After + public void closeTester() { + tester.close(); + } + + /** This test gives examples of the node requests that can be made to nodes/v2 */ + @Test + public void test_requests() throws Exception { + // GET + assertFile(new Request("http://localhost:8080/nodes/v2/"), "root.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/state/"), "states.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/state/?recursive=true"), "states-recursive.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/state/active?recursive=true"), "active-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/"), "nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true"), "nodes-recursive.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "node2.json"); + + // GET with filters + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host6.yahoo.com%20host2.yahoo.com"), "application2-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "content-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterId=id2"), "application2-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&application=tenant2.application2.instance2"), "application2-nodes.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=dockerhost1.yahoo.com"), "child-nodes.json"); + + // POST restart command + assertRestart(1, new Request("http://localhost:8080/nodes/v2/command/restart?hostname=host2.yahoo.com", + new byte[0], Request.Method.POST)); + assertRestart(2, new Request("http://localhost:8080/nodes/v2/command/restart?application=tenant2.application2.instance2", + new byte[0], Request.Method.POST)); + assertRestart(11, new Request("http://localhost:8080/nodes/v2/command/restart", + new byte[0], Request.Method.POST)); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), + "\"restartGeneration\":3"); + + // POST reboot command + assertReboot(12, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", + new byte[0], Request.Method.POST)); + assertReboot(2, new Request("http://localhost:8080/nodes/v2/command/reboot?application=tenant2.application2.instance2", + new byte[0], Request.Method.POST)); + assertReboot(19, new Request("http://localhost:8080/nodes/v2/command/reboot", + new byte[0], Request.Method.POST)); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), + "\"rebootGeneration\":4"); + + // POST new nodes + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.8.1") + "," + // test with only 1 ip address + asNodeJson("host9.yahoo.com", "large-variant", "127.0.9.1", "::9:1") + "," + + asHostJson("parent2.yahoo.com", "large-variant", Optional.of(TenantName.from("myTenant")), "127.0.127.1", "::127:1") + "," + + asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com", "::11") + "]"). + getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 4 nodes to the provisioned state\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), "node8.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com"), "node9.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com"), "node11.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/parent2.yahoo.com"), "parent2.json"); + + // POST duplicate node + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.8") + "]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); + + // DELETE a provisioned node + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Removed host9.yahoo.com\"}"); + + // PUT nodes ready + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host8.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host8.yahoo.com to dirty\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host8.yahoo.com to ready\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), + "\"state\":\"ready\""); + // calling ready again is a noop: + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host8.yahoo.com to ready\"}"); + + // PUT a node in failed ... + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host2.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host2.yahoo.com to failed\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), + "\"state\":\"failed\""); + // ... and put it back in active (after fixing). This is useful to restore data when multiple nodes fail. + assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host2.yahoo.com to active\"}"); + + // PUT a node in parked ... + assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host8.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host8.yahoo.com to parked\"}"); + tester.assertResponseContains(new Request("http://localhost:8080()/nodes/v2/node/host8.yahoo.com"), + "\"state\":\"parked\""); + // ... and delete it + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Removed host8.yahoo.com\"}"); + + // or, PUT a node in failed ... + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/test-node-pool-102-2", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved test-node-pool-102-2 to failed\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), + "\"state\":\"failed\""); + // ... and deallocate it such that it moves to dirty and is recycled + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/test-node-pool-102-2", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved test-node-pool-102-2 to dirty\"}"); + + // ... and set it back to ready as if this was from the node-admin with the temporary state rest api + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/test-node-pool-102-2", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved test-node-pool-102-2 to ready\"}"); + + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", new byte[0], Request.Method.GET), + 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'test-node-pool-102-2'\"}"); + + // Put a host in failed and make sure it's children are also failed + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/dockerhost1.yahoo.com", new byte[0], Request.Method.PUT), + "{\"message\":\"Moved dockerhost1.yahoo.com, host4.yahoo.com to failed\"}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed"), "{\"nodes\":[" + + "{\"url\":\"http://localhost:8080/nodes/v2/node/host5.yahoo.com\"}," + + "{\"url\":\"http://localhost:8080/nodes/v2/node/host4.yahoo.com\"}," + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}]}"); + + // Update (PATCH) a node (multiple fields can also be sent in one request body) + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentRebootGeneration\": 1}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"flavor\": \"d-2-8-100\"}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentVespaVersion\": \"5.104.142\"}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"parentHostname\": \"parent.yahoo.com\"}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"ipAddresses\": [\"127.0.0.1\",\"::1\"]}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"wantToRetire\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentVespaVersion\": \"6.43.0\",\"currentDockerImage\": \"docker-registry.domain.tld:8080/dist/vespa:6.45.0\"}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"openStackId\": \"patched-openstackid\"}"), Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"modelName\": null}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "modelName", false); + tester.container().handleRequest((new Request("http://localhost:8080/nodes/v2/upgrade/tenant", Utf8.toBytes("{\"dockerImage\": \"docker.domain.tld/my/image\"}"), Request.Method.PATCH))); + + ((OrchestratorMock) tester.container().components().getComponent(OrchestratorMock.class.getName())) + .suspend(new HostName("host4.yahoo.com")); + + assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); + } + + @Test + public void test_application_requests() throws Exception { + assertFile(new Request("http://localhost:8080/nodes/v2/application/"), "applications.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/application/tenant1.application1.instance1"), + "application1.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/application/tenant2.application2.instance2"), + "application2.json"); + } + + @Test + public void maintenance_requests() throws Exception { + // POST deactivation of a maintenance job + assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/inactive/NodeFailer", + new byte[0], Request.Method.POST), + "{\"message\":\"Deactivated job 'NodeFailer'\"}"); + // GET a list of all maintenance jobs + assertFile(new Request("http://localhost:8080/nodes/v2/maintenance/"), "maintenance.json"); + + // DELETE deactivation of a maintenance job + assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/inactive/NodeFailer", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Re-activated job 'NodeFailer'\"}"); + + // POST run of a maintenance job + assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/run/PeriodicApplicationMaintainer", + new byte[0], Request.Method.POST), + "{\"message\":\"Executed job 'PeriodicApplicationMaintainer'\"}"); + + // POST run of unknown maintenance job + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/run/foo", + new byte[0], Request.Method.POST), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such job 'foo'\"}"); + } + + @Test + public void post_with_patch_method_override_in_header_is_handled_as_patch() throws Exception { + Request req = new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.POST); + req.getHeaders().add("X-HTTP-Method-Override", "PATCH"); + assertResponse(req, "{\"message\":\"Updated host4.yahoo.com\"}"); + } + + @Test + public void post_with_invalid_method_override_in_header_gives_sane_error_message() throws Exception { + Request req = new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.POST); + req.getHeaders().add("X-HTTP-Method-Override", "GET"); + tester.assertResponse(req, 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Illegal X-HTTP-Method-Override header for POST request. Accepts 'PATCH' but got 'GET'\"}"); + } + + @Test + public void post_node_with_ip_address() throws Exception { + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("ipv4-host.yahoo.com", "default","127.0.0.1") + "]"). + getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("ipv6-host.yahoo.com", "default", "::1") + "]"). + getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("dual-stack-host.yahoo.com", "default", "127.0.254.254", "::254:254") + "]"). + getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + } + + @Test + public void post_node_with_duplicate_ip_address() throws Exception { + Request req = new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("host-with-ip.yahoo.com", "default", "foo") + "]"). + getBytes(StandardCharsets.UTF_8), + Request.Method.POST); + tester.assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Found one or more invalid addresses in [foo]: 'foo' is not an IP string literal.\"}"); + + // Attempt to POST tenant node with already assigned IP + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asNodeJson("tenant-node-foo.yahoo.com", "default", "127.0.1.1") + "]", + Request.Method.POST), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.1.1] to tenant-node-foo.yahoo.com: [127.0.1.1] already assigned to host1.yahoo.com\"}"); + + // Attempt to PATCH existing tenant node with already assigned IP + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", + "{\"ipAddresses\": [\"127.0.2.1\"]}", + Request.Method.PATCH), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [127.0.2.1] to test-node-pool-102-2: [127.0.2.1] already assigned to host2.yahoo.com\"}"); + + // Attempt to POST host node with already assigned IP + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asHostJson("host200.yahoo.com", "default", Optional.empty(), "127.0.2.1") + "]", + Request.Method.POST), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.2.1] to host200.yahoo.com: [127.0.2.1] already assigned to host2.yahoo.com\"}"); + + // Attempt to PATCH host node with IP in the pool of another node + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "{\"ipAddresses\": [\"::104:3\"]}", + Request.Method.PATCH), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [::100:4, ::100:3, ::100:2, ::104:3] to dockerhost1.yahoo.com: [::104:3] already assigned to dockerhost5.yahoo.com\"}"); + + // Node types running a single container can share their IP address with child node + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.42.1") + "]", + Request.Method.POST), 200, + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asDockerNodeJson("cfg42.yahoo.com", NodeType.config, "cfghost42.yahoo.com", "127.0.42.1") + "]", + Request.Method.POST), 200, + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + + // ... but cannot share with child node of wrong type + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asDockerNodeJson("proxy42.yahoo.com", NodeType.proxy, "cfghost42.yahoo.com", "127.0.42.1") + "]", + Request.Method.POST), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.42.1] to proxy42.yahoo.com: [127.0.42.1] already assigned to cfg42.yahoo.com\"}"); + + // ... nor with child node on different host + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.43.1") + "]", + Request.Method.POST), 200, + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", + "{\"ipAddresses\": [\"127.0.43.1\"]}", + Request.Method.PATCH), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [127.0.43.1] to cfg42.yahoo.com: [127.0.43.1] already assigned to cfghost43.yahoo.com\"}"); + } + + @Test + public void post_controller_node() throws Exception { + String data = "[{\"hostname\":\"controller1.yahoo.com\", \"openStackId\":\"fake-controller1.yahoo.com\"," + + createIpAddresses("127.0.0.1") + + "\"flavor\":\"default\"" + + ", \"type\":\"controller\"}]"; + assertResponse(new Request("http://localhost:8080/nodes/v2/node", data.getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + + assertFile(new Request("http://localhost:8080/nodes/v2/node/controller1.yahoo.com"), "controller1.json"); + } + + @Test + public void fails_to_ready_node_with_hard_fail() throws Exception { + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("host12.yahoo.com", "default") + "]"). + getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + String msg = "Actual disk space (2TB) differs from spec (3TB)"; + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host12.yahoo.com", + Utf8.toBytes("{\"reports\":{\"diskSpace\":{\"createdMillis\":2,\"description\":\"" + msg + "\",\"type\": \"HARD_FAIL\"}}}"), + Request.Method.PATCH), + "{\"message\":\"Updated host12.yahoo.com\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host12.yahoo.com", new byte[0], Request.Method.PUT), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"provisioned host host12.yahoo.com cannot be readied because it has " + + "hard failures: [diskSpace reported 1970-01-01T00:00:00.002Z: " + msg + "]\"}"); + } + + @Test + public void patching_dirty_node_does_not_increase_reboot_generation() throws Exception { + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("foo.yahoo.com", "default") + "]"). + getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/foo.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved foo.yahoo.com to failed\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/foo.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved foo.yahoo.com to dirty\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), + "\"rebootGeneration\":1"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com", + Utf8.toBytes("{\"currentRebootGeneration\": 42}"), Request.Method.PATCH), + "{\"message\":\"Updated foo.yahoo.com\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), + "\"rebootGeneration\":1"); + } + + @Test + public void acl_request_by_tenant_node() throws Exception { + String hostname = "foo.yahoo.com"; + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson(hostname, "default", "127.0.222.1") + "]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/" + hostname, + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved foo.yahoo.com to dirty\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/" + hostname, + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved foo.yahoo.com to ready\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/acl/" + hostname), "acl-tenant-node.json"); + } + + @Test + public void acl_request_by_config_server() throws Exception { + assertFile(new Request("http://localhost:8080/nodes/v2/acl/cfg1.yahoo.com"), "acl-config-server.json"); + } + + @Test + public void test_invalid_requests() throws Exception { + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/node-does-not-exist", + new byte[0], Request.Method.GET), + 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'node-does-not-exist'\"}"); + + // Attempt to fail and ready an allocated node without going through dirty + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/node-does-not-exist", + new byte[0], Request.Method.PUT), + 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not move node-does-not-exist to failed: Node not found\"}"); + + // Attempt to fail and ready an allocated node without going through dirty + assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host1.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host1.yahoo.com to failed\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com", + new byte[0], Request.Method.PUT), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make failed host host1.yahoo.com allocated to tenant1.application1.instance1 as 'container/id1/0/0' available for new allocation as it is not in state [dirty]\"}"); + + // (... while dirty then ready works (the ready move will be initiated by node maintenance)) + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host1.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host1.yahoo.com to dirty\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host1.yahoo.com to ready\"}"); + + // Attempt to park and ready an allocated node without going through dirty + assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host2.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host2.yahoo.com to parked\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com", + new byte[0], Request.Method.PUT), + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make parked host host2.yahoo.com allocated to tenant2.application2.instance2 as 'content/id2/0/0' available for new allocation as it is not in state [dirty]\"}"); + // (... while dirty then ready works (the ready move will be initiated by node maintenance)) + assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host2.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host2.yahoo.com to dirty\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com", + new byte[0], Request.Method.PUT), + "{\"message\":\"Moved host2.yahoo.com to ready\"}"); + + // Attempt to DELETE a node which has been removed + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com", + new byte[0], Request.Method.DELETE), + 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'host2.yahoo.com'\"}"); + + // Attempt to DELETE allocated node + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + new byte[0], Request.Method.DELETE), + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"active child node host4.yahoo.com allocated to tenant3.application3.instance3 as 'content/id3/0/0' is currently allocated and cannot be removed\"}"); + + // PUT current restart generation with string instead of long + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": \"1\"}"), Request.Method.PATCH), + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Expected a LONG value, got a STRING\"}"); + + // PUT flavor with long instead of string + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"flavor\": 1}"), Request.Method.PATCH), + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'flavor': Expected a STRING value, got a LONG\"}"); + + // Attempt to set nonexisting node to active + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", + new byte[0], Request.Method.PUT), 404, + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not move host2.yahoo.com to active: Node not found\"}"); + + // Attempt to POST duplicate nodes + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.1", "::254:1") + "," + + asNodeJson("host8.yahoo.com", "large-variant", "127.0.253.1", "::253:1") + "]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add nodes: provisioned host host8.yahoo.com is duplicated in the argument list\"}"); + + // Attempt to PATCH field not relevant for child node + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", + Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'modelName': A child node cannot have model name set\"}"); + } + + @Test + public void test_node_patching() throws Exception { + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{" + + "\"currentRestartGeneration\": 1," + + "\"currentRebootGeneration\": 3," + + "\"flavor\": \"medium-disk\"," + + "\"currentVespaVersion\": \"5.104.142\"," + + "\"failCount\": 0," + + "\"parentHostname\": \"parent.yahoo.com\"" + + "}" + ), + Request.Method.PATCH), + "{\"message\":\"Updated host4.yahoo.com\"}"); + + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/doesnotexist.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), + Request.Method.PATCH), + 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node found with hostname doesnotexist.yahoo.com\"}"); + + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Node is not allocated\"}"); + } + + @Test + public void test_node_patch_to_remove_docker_ready_fields() throws Exception { + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com", + Utf8.toBytes("{" + + "\"currentVespaVersion\": \"\"," + + "\"currentDockerImage\": \"\"" + + "}" + ), + Request.Method.PATCH), + "{\"message\":\"Updated host5.yahoo.com\"}"); + + assertFile(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com"), "node5-after-changes.json"); + } + + @Test + public void test_reports_patching() throws IOException { + // Add report + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + Utf8.toBytes("{" + + " \"reports\": {" + + " \"actualCpuCores\": {" + + " \"createdMillis\": 1, " + + " \"description\": \"Actual number of CPU cores (2) differs from spec (4)\"," + + " \"type\": \"HARD_FAIL\"," + + " \"value\":2" + + " }," + + " \"diskSpace\": {" + + " \"createdMillis\": 2, " + + " \"description\": \"Actual disk space (2TB) differs from spec (3TB)\"," + + " \"type\": \"HARD_FAIL\"," + + " \"details\": {" + + " \"inGib\": 3," + + " \"disks\": [\"/dev/sda1\", \"/dev/sdb3\"]" + + " }" + + " }" + + " }" + + "}"), + Request.Method.PATCH), + "{\"message\":\"Updated host6.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); + + // Patching with an empty reports is no-op + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + Utf8.toBytes("{\"reports\": {}}"), + Request.Method.PATCH), + 200, + "{\"message\":\"Updated host6.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); + + // Patching existing report overwrites + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + Utf8.toBytes("{" + + " \"reports\": {" + + " \"actualCpuCores\": {" + + " \"createdMillis\": 3 " + + " }" + + " }" + + "}"), + Request.Method.PATCH), + 200, + "{\"message\":\"Updated host6.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports-2.json"); + + // Clearing one report + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + Utf8.toBytes("{\"reports\": { \"diskSpace\": null } }"), + Request.Method.PATCH), + "{\"message\":\"Updated host6.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports-3.json"); + + // Clearing all reports + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + Utf8.toBytes("{\"reports\": null }"), + Request.Method.PATCH), + "{\"message\":\"Updated host6.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6.json"); + } + + @Test + public void test_upgrade() throws IOException { + // Initially, no versions are set + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{},\"osVersions\":{},\"dockerImages\":{}}"); + + // Set version for config, confighost and controller + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", + Utf8.toBytes("{\"version\": \"6.123.456\"}"), + Request.Method.PATCH), + "{\"message\":\"Set version to 6.123.456 for nodes of type config\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": \"6.123.456\"}"), + Request.Method.PATCH), + "{\"message\":\"Set version to 6.123.456 for nodes of type confighost\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/controller", + Utf8.toBytes("{\"version\": \"6.123.456\"}"), + Request.Method.PATCH), + "{\"message\":\"Set version to 6.123.456 for nodes of type controller\"}"); + + + // Verify versions are set + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); + + // Setting version for unsupported node type fails + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/tenant", + Utf8.toBytes("{\"version\": \"6.123.456\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Target version for type tenant is not allowed\"}"); + + // Omitting version field fails + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"At least one of 'version', 'osVersion' or 'dockerImage' must be set\"}"); + + // Downgrade without force fails + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": \"6.123.1\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. " + + "Current target version: 6.123.456, attempted to set target version: 6.123.1\"}"); + + // Downgrade with force is OK + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": \"6.123.1\",\"force\":true}"), + Request.Method.PATCH), + "{\"message\":\"Set version to 6.123.1 for nodes of type confighost\"}"); + + // Verify version has been updated + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); + + // Setting empty version without force fails + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": null}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. Current target version: 6.123.1, attempted to set target version: 0.0.0\"}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": null, \"force\": true}"), + Request.Method.PATCH), + "{\"message\":\"Set version to 0.0.0 for nodes of type confighost\"}"); + + // Verify version has been removed + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), + "{\"versions\":{\"config\":\"6.123.456\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); + + // Upgrade OS for confighost and host + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.5.2 for nodes of type confighost\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}"); + + // OS versions are set + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), + "{\"versions\":{\"config\":\"6.123.456\",\"controller\":\"6.123.456\"},\"osVersions\":{\"host\":\"7.5.2\",\"confighost\":\"7.5.2\"},\"dockerImages\":{}}"); + + // Upgrade OS and Vespa together + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": \"6.124.42\", \"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set version to 6.124.42, osVersion to 7.5.2 for nodes of type confighost\"}"); + + // Attempt to upgrade unsupported node type + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Node type 'config' does not support OS upgrades\"}"); + + // Attempt to downgrade OS + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": \"7.4.2\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set target OS version to 7.4.2 without setting 'force', as it's lower than the current version: 7.5.2\"}"); + + // Downgrading OS with force succeeds + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": \"7.4.2\", \"force\": true}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.4.2 for nodes of type confighost\"}"); + + // Current target is considered bad, remove it + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": null}"), + Request.Method.PATCH), + 200, + "{\"message\":\"Set osVersion to null for nodes of type confighost\"}"); + + // Set docker image for config and tenant + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/tenant", + Utf8.toBytes("{\"dockerImage\": \"my-repo.my-domain.example:1234/repo/tenant\"}"), + Request.Method.PATCH), + "{\"message\":\"Set docker image to my-repo.my-domain.example:1234/repo/tenant for nodes of type tenant\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", + Utf8.toBytes("{\"dockerImage\": \"my-repo.my-domain.example:1234/repo/image\"}"), + Request.Method.PATCH), + "{\"message\":\"Set docker image to my-repo.my-domain.example:1234/repo/image for nodes of type config\"}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), + "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.124.42\",\"controller\":\"6.123.456\"},\"osVersions\":{\"host\":\"7.5.2\"},\"dockerImages\":{\"tenant\":\"my-repo.my-domain.example:1234/repo/tenant\",\"config\":\"my-repo.my-domain.example:1234/repo/image\"}}"); + + // Cannot set docker image for non docker node type + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"dockerImage\": \"my-repo.my-domain.example:1234/repo/image\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Setting docker image for confighost nodes is unsupported\"}"); + } + + @Test + public void test_os_version() throws Exception { + // Schedule OS upgrade + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}"); + + // Activate target + var nodeRepository = (NodeRepository)tester.container().components().getComponent(MockNodeRepository.class.getName()); + var osUpgradeActivator = new OsUpgradeActivator(nodeRepository, Duration.ofDays(1)); + osUpgradeActivator.run(); + + // Other node type does not return wanted OS version + Response r = tester.container().handleRequest(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com")); + assertFalse("Response omits wantedOsVersions field", r.getBodyAsString().contains("wantedOsVersion")); + + // Node updates its node object after upgrading OS + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-os-upgrade-complete.json"); + + // Another node upgrades + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", + Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Updated dockerhost2.yahoo.com\"}"); + + // Filter nodes by osVersion + assertResponse(new Request("http://localhost:8080/nodes/v2/node/?osVersion=7.5.2"), + "{\"nodes\":[" + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com\"}," + + "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}" + + "]}"); + } + + @Test + public void test_firmware_upgrades() throws IOException { + // dockerhost1 checks firmware at time 100. + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"currentFirmwareCheck\":100}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + + // Schedule a firmware check at time 123 (the mock default). + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/firmware", new byte[0], Request.Method.POST), + "{\"message\":\"Will request firmware checks on all hosts.\"}"); + + // dockerhost1 displays both values. + assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), + "dockerhost1-with-firmware-data.json"); + + // host1 has no wantedFirmwareCheck, as it's not a docker host. + assertFile(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), + "node1.json"); + + // Cancel the firmware check. + assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/firmware", new byte[0], Request.Method.DELETE), + "{\"message\":\"Cancelled outstanding requests for firmware checks\"}"); + } + + @Test + public void test_capacity() throws Exception { + assertFile(new Request("http://localhost:8080/nodes/v2/capacity/?json=true"), "capacity-zone.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/capacity?json=true"), "capacity-zone.json"); + + List hostsToRemove = List.of( + "dockerhost1.yahoo.com", + "dockerhost2.yahoo.com", + "dockerhost3.yahoo.com", + "dockerhost4.yahoo.com" + ); + String requestUriTemplate = "http://localhost:8080/nodes/v2/capacity/?json=true&hosts=%s"; + + assertFile(new Request(String.format(requestUriTemplate, + String.join(",", hostsToRemove.subList(0, 3)))), + "capacity-hostremoval-possible.json"); + assertFile(new Request(String.format(requestUriTemplate, + String.join(",", hostsToRemove))), + "capacity-hostremoval-impossible.json"); + } + + + /** Tests the rendering of each node separately to make it easier to find errors */ + @Test + public void test_single_node_rendering() throws Exception { + for (int i = 1; i <= 14; i++) { + if (i == 8 || i == 9 || i == 11 || i == 12) continue; // these nodes are added later + assertFile(new Request("http://localhost:8080/nodes/v2/node/host" + i + ".yahoo.com"), "node" + i + ".json"); + } + } + + @Test + public void test_flavor_overrides() throws Exception { + String host = "parent2.yahoo.com"; + // Test adding with overrides + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[{\"hostname\":\"" + host + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + + "\"flavor\":\"large-variant\",\"resources\":{\"diskGb\":1234,\"memoryGb\":4321}}]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can only override disk GB for configured flavor\"}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[{\"hostname\":\"" + host + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + + "\"flavor\":\"large-variant\",\"type\":\"host\",\"resources\":{\"diskGb\":1234}}]"). + getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), + "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":1234.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); + + // Test adding tenant node + String tenant = "node-1-3.yahoo.com"; + String resources = "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":1234.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"slow\",\"storageType\":\"remote\"}"; + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[{\"hostname\":\"" + tenant + "\"," + createIpAddresses("::2") + "\"openStackId\":\"osid-124\"," + + "\"type\":\"tenant\"," + resources + "}]"). + getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + tenant), resources); + + // Test patching with overrides + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, + "{\"minDiskAvailableGb\":5432,\"minMainMemoryAvailableGb\":2345}".getBytes(StandardCharsets.UTF_8), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'minMainMemoryAvailableGb': Can only override disk GB for configured flavor\"}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, + "{\"minDiskAvailableGb\":5432}".getBytes(StandardCharsets.UTF_8), + Request.Method.PATCH), + "{\"message\":\"Updated " + host + "\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), + "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":5432.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); + } + + @Test + public void test_node_resources() throws Exception { + String hostname = "node123.yahoo.com"; + String resources = "\"resources\":{\"vcpu\":5.0,\"memoryGb\":4321.0,\"diskGb\":1234.0,\"bandwidthGbps\":0.3,\"diskSpeed\":\"slow\",\"storageType\":\"local\"}"; + // Test adding new node with resources + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[{\"hostname\":\"" + hostname + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + + resources.replace("\"memoryGb\":4321.0,", "") + "}]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Required field 'memoryGb' is missing\"}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[{\"hostname\":\"" + hostname + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + resources + "}]") + .getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + "{\"message\":\"Added 1 nodes to the provisioned state\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), resources); + + // Test patching with overrides + assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname, + "{\"diskGb\":12,\"memoryGb\":34,\"vcpu\":56,\"fastDisk\":true,\"remoteStorage\":true,\"bandwidthGbps\":78.0}".getBytes(StandardCharsets.UTF_8), + Request.Method.PATCH), + "{\"message\":\"Updated " + hostname + "\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), + "\"resources\":{\"vcpu\":56.0,\"memoryGb\":34.0,\"diskGb\":12.0,\"bandwidthGbps\":78.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); + } + + private static String asDockerNodeJson(String hostname, String parentHostname, String... ipAddress) { + return asDockerNodeJson(hostname, NodeType.tenant, parentHostname, ipAddress); + } + + private static String asDockerNodeJson(String hostname, NodeType nodeType, String parentHostname, String... ipAddress) { + return "{\"hostname\":\"" + hostname + "\", \"parentHostname\":\"" + parentHostname + "\"," + + createIpAddresses(ipAddress) + + "\"openStackId\":\"" + hostname + "\",\"flavor\":\"d-1-1-100\"" + + (nodeType != NodeType.tenant ? ",\"type\":\"" + nodeType + "\"" : "") + + "}"; + } + + private static String asNodeJson(String hostname, String flavor, String... ipAddress) { + return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\"," + + createIpAddresses(ipAddress) + + "\"flavor\":\"" + flavor + "\"}"; + } + + private static String asHostJson(String hostname, String flavor, Optional reservedTo, String... ipAddress) { + return asNodeJson(hostname, NodeType.host, flavor, reservedTo, ipAddress); + } + + private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional reservedTo, String... ipAddress) { + return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\"," + + createIpAddresses(ipAddress) + + "\"flavor\":\"" + flavor + "\"" + + (reservedTo.isPresent() ? ", \"reservedTo\":\"" + reservedTo.get().value() + "\"" : "") + + ", \"type\":\"" + nodeType + "\"}"; + } + + private static String createIpAddresses(String... ipAddress) { + return "\"ipAddresses\":[" + + Arrays.stream(ipAddress) + .map(ip -> "\"" + ip + "\"") + .collect(Collectors.joining(",")) + + "],"; + } + + private void assertRestart(int restartCount, Request request) throws IOException { + tester.assertResponse(request, 200, "{\"message\":\"Scheduled restart of " + restartCount + " matching nodes\"}"); + } + + private void assertReboot(int rebootCount, Request request) throws IOException { + tester.assertResponse(request, 200, "{\"message\":\"Scheduled reboot of " + rebootCount + " matching nodes\"}"); + } + + private void assertFile(Request request, String file) throws IOException { + tester.assertFile(request, file); + } + + private void assertResponse(Request request, String file) throws IOException { + tester.assertResponse(request, file); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java new file mode 100644 index 00000000000..847a4e2500b --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java @@ -0,0 +1,96 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.JDisc; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; +import org.junit.ComparisonFailure; + +import java.io.File; +import java.io.IOException; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class RestApiTester { + + private final static String responsesPath = "src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/"; + + private final JDisc container; + + public RestApiTester() { + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); + } + + public void close() { + if (container != null) container.close(); + } + + public JDisc container() { return container; } + + /** Asserts a particular response and 200 as response status */ + public void assertResponse(Request request, String responseMessage) throws IOException { + assertResponse(request, 200, responseMessage); + } + + public void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException { + Response response = container.handleRequest(request); + // Compare both status and message at once for easier diagnosis + assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage, + "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString()); + } + + public void assertResponseContains(Request request, String responseSnippet) throws IOException { + assertPartialResponse(request, responseSnippet, true); + } + + public void assertPartialResponse(Request request, String responseSnippet, boolean match) throws IOException { + String response = container.handleRequest(request).getBodyAsString(); + assertEquals(String.format("Expected response to " + (match ? " " : "not ") + "contain: %s\nResponse: %s", + responseSnippet, response), match, response.contains(responseSnippet)); + } + + public void assertFile(Request request, String responseFile) throws IOException { + String expectedResponse = IOUtils.readFile(new File(responsesPath + responseFile)); + expectedResponse = include(expectedResponse); + expectedResponse = expectedResponse.replaceAll("(\"[^\"]*\")|\\s*", "$1"); // Remove whitespace + String responseString = container.handleRequest(request).getBodyAsString(); + if (expectedResponse.contains("(ignore)")) { + // Convert expected response to a literal pattern and replace any ignored field with a pattern that matches + // until the first stop character + String stopCharacters = "[^,:\\\\[\\\\]{}]"; + String expectedResponsePattern = Pattern.quote(expectedResponse) + .replaceAll("\"?\\(ignore\\)\"?", "\\\\E" + + stopCharacters + "*\\\\Q"); + if (!Pattern.matches(expectedResponsePattern, responseString)) { + throw new ComparisonFailure(responseFile + " (with ignored fields)", expectedResponsePattern, + responseString); + } + } else { + assertEquals(responseFile, expectedResponse, responseString); + } + } + + /** Replaces @include(localFile) with the content of the file */ + public String include(String response) throws IOException { + // Please don't look at this code + int includeIndex = response.indexOf("@include("); + if (includeIndex < 0) return response; + String prefix = response.substring(0, includeIndex); + String rest = response.substring(includeIndex + "@include(".length()); + int filenameEnd = rest.indexOf(")"); + String includeFileName = rest.substring(0, filenameEnd); + String includedContent = IOUtils.readFile(new File(responsesPath + includeFileName)); + includedContent = include(includedContent); + String postFix = rest.substring(filenameEnd + 1); + postFix = include(postFix); + return prefix + includedContent + postFix; + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json new file mode 100644 index 00000000000..55891309856 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json @@ -0,0 +1,237 @@ +{ + "trustedNodes": [ + { + "hostname": "cfg1.yahoo.com", + "type": "config", + "ipAddress": "127.0.201.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "cfg1.yahoo.com", + "type": "config", + "ipAddress": "::201:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "cfg2.yahoo.com", + "type": "config", + "ipAddress": "127.0.202.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "cfg2.yahoo.com", + "type": "config", + "ipAddress": "::202:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "dockerhost1.yahoo.com", + "type": "host", + "ipAddress": "127.0.100.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "dockerhost1.yahoo.com", + "type": "host", + "ipAddress": "::100:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "dockerhost2.yahoo.com", + "type": "host", + "ipAddress": "127.0.101.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "dockerhost2.yahoo.com", + "type": "host", + "ipAddress": "::101:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "dockerhost3.yahoo.com", + "type": "host", + "ipAddress": "127.0.102.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "dockerhost3.yahoo.com", + "type": "host", + "ipAddress": "::102:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "dockerhost4.yahoo.com", + "type": "host", + "ipAddress": "127.0.103.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "dockerhost4.yahoo.com", + "type": "host", + "ipAddress": "::103:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "dockerhost5.yahoo.com", + "type": "host", + "ipAddress": "127.0.104.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "dockerhost5.yahoo.com", + "type": "host", + "ipAddress": "::104:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host1.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.1.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host1.yahoo.com", + "type": "tenant", + "ipAddress": "::1:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host10.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.10.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host10.yahoo.com", + "type": "tenant", + "ipAddress": "::10:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host13.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.13.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host13.yahoo.com", + "type": "tenant", + "ipAddress": "::13:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host14.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.14.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host14.yahoo.com", + "type": "tenant", + "ipAddress": "::14:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host2.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.2.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host2.yahoo.com", + "type": "tenant", + "ipAddress": "::2:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host3.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.3.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host3.yahoo.com", + "type": "tenant", + "ipAddress": "::3:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host4.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.4.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host4.yahoo.com", + "type": "tenant", + "ipAddress": "::4:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host5.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.5.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host5.yahoo.com", + "type": "tenant", + "ipAddress": "::5:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host55.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.55.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host55.yahoo.com", + "type": "tenant", + "ipAddress": "::55:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host6.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.6.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host6.yahoo.com", + "type": "tenant", + "ipAddress": "::6:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host7.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.7.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host7.yahoo.com", + "type": "tenant", + "ipAddress": "::7:1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "test-node-pool-102-2", + "type": "tenant", + "ipAddress": "::102:2", + "trustedBy": "cfg1.yahoo.com" + } + ], + "trustedNetworks": [], + "trustedPorts": [ + { + "port":22, + "trustedBy":"cfg1.yahoo.com" + }, + { + "port": 4443, + "trustedBy": "cfg1.yahoo.com" + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-tenant-node.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-tenant-node.json new file mode 100644 index 00000000000..16cf5c36dcc --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-tenant-node.json @@ -0,0 +1,179 @@ +{ + "trustedNodes": [ + { + "hostname": "cfg1.yahoo.com", + "type": "config", + "ipAddress": "127.0.201.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "cfg1.yahoo.com", + "type": "config", + "ipAddress": "::201:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "cfg2.yahoo.com", + "type": "config", + "ipAddress": "127.0.202.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "cfg2.yahoo.com", + "type": "config", + "ipAddress": "::202:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "foo.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.222.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host1.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.1.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host1.yahoo.com", + "type": "tenant", + "ipAddress": "::1:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host10.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.10.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host10.yahoo.com", + "type": "tenant", + "ipAddress": "::10:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host13.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.13.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host13.yahoo.com", + "type": "tenant", + "ipAddress": "::13:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host14.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.14.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host14.yahoo.com", + "type": "tenant", + "ipAddress": "::14:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host2.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.2.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host2.yahoo.com", + "type": "tenant", + "ipAddress": "::2:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host3.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.3.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host3.yahoo.com", + "type": "tenant", + "ipAddress": "::3:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host4.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.4.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host4.yahoo.com", + "type": "tenant", + "ipAddress": "::4:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host5.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.5.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host5.yahoo.com", + "type": "tenant", + "ipAddress": "::5:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host55.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.55.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host55.yahoo.com", + "type": "tenant", + "ipAddress": "::55:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host6.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.6.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host6.yahoo.com", + "type": "tenant", + "ipAddress": "::6:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host7.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.7.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host7.yahoo.com", + "type": "tenant", + "ipAddress": "::7:1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "test-node-pool-102-2", + "type": "tenant", + "ipAddress": "::102:2", + "trustedBy": "foo.yahoo.com" + } + ], + "trustedNetworks": [], + "trustedPorts": [ + { + "port":22, + "trustedBy":"foo.yahoo.com" + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json new file mode 100644 index 00000000000..c61a755d989 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json @@ -0,0 +1,15 @@ +{ + "nodes": [ + @include(node6.json), + @include(docker-container1.json), + @include(node13.json), + @include(node2.json), + @include(node14.json), + @include(node4.json), + @include(docker-node4.json), + @include(docker-node5.json), + @include(docker-node2.json), + @include(docker-node1.json), + @include(docker-node3.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json new file mode 100644 index 00000000000..ce2309e4d6c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json @@ -0,0 +1,56 @@ +{ + "url" : "http://localhost:8080/nodes/v2/applications/tenant1.application1.instance1", + "id" : "tenant1.application1.instance1", + "clusters" : { + "id1" : { + "min" : { + "nodes" : 2, + "groups" : 1, + "resources" : { + "vcpu" : 2.0, + "memoryGb" : 8.0, + "diskGb" : 50.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + }, + "max" : { + "nodes" : 8, + "groups" : 2, + "resources" : { + "vcpu" : 4.0, + "memoryGb" : 16.0, + "diskGb" : 1000.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + }, + "suggested" : { + "nodes" : 6, + "groups" : 2, + "resources" : { + "vcpu" : 3.0, + "memoryGb" : 20.0, + "diskGb" : 100.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + }, + "target" : { + "nodes" : 4, + "groups" : 1, + "resources" : { + "vcpu" : 3.0, + "memoryGb" : 16.0, + "diskGb" : 100.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + } + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json new file mode 100644 index 00000000000..4581ecba73d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json @@ -0,0 +1,6 @@ +{ + "nodes": [ + @include(node6.json), + @include(node2.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json new file mode 100644 index 00000000000..75af5d4a328 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json @@ -0,0 +1,32 @@ +{ + "url": "http://localhost:8080/nodes/v2/applications/tenant2.application2.instance2", + "id": "tenant2.application2.instance2", + "clusters": { + "id2": { + "min": { + "nodes": 2, + "groups": 1, + "resources": { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "fast", + "storageType": "any" + } + }, + "max": { + "nodes": 2, + "groups": 1, + "resources": { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "fast", + "storageType": "any" + } + } + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/applications.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/applications.json new file mode 100644 index 00000000000..6d25b3e59f5 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/applications.json @@ -0,0 +1,20 @@ +{ + "applications" : [ + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant4.application4.instance4", + "id" : "tenant4.application4.instance4" + }, + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant2.application2.instance2", + "id" : "tenant2.application2.instance2" + }, + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant3.application3.instance3", + "id" : "tenant3.application3.instance3" + }, + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant1.application1.instance1", + "id" : "tenant1.application1.instance1" + } + ] +} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-impossible.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-impossible.json new file mode 100644 index 00000000000..f85e0dcc270 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-impossible.json @@ -0,0 +1,20 @@ +{ + "hostsToRemove": [ + "dockerhost1.yahoo.com", + "dockerhost2.yahoo.com", + "dockerhost3.yahoo.com", + "dockerhost4.yahoo.com" + ], + "removalPossible": false, + "history": [ + { + "tenant": "host4.yahoo.com", + "newParent": "dockerhost5.yahoo.com", + "eligibleParents": 1 + }, + { + "tenant": "test-node-pool-102-2", + "eligibleParents": 0 + } + ] +} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-possible.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-possible.json new file mode 100644 index 00000000000..177e1f317f7 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-possible.json @@ -0,0 +1,20 @@ +{ + "hostsToRemove": [ + "dockerhost1.yahoo.com", + "dockerhost2.yahoo.com", + "dockerhost3.yahoo.com" + ], + "removalPossible": true, + "history": [ + { + "tenant": "host4.yahoo.com", + "newParent": "dockerhost4.yahoo.com", + "eligibleParents": 1 + }, + { + "tenant": "test-node-pool-102-2", + "newParent": "dockerhost5.yahoo.com", + "eligibleParents": 1 + } + ] +} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json new file mode 100644 index 00000000000..f6b2f96023a --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json @@ -0,0 +1,46 @@ +{ + "totalHosts": 5, + "couldLoseHosts": 4, + "failedTenantParent": "dockerhost1.yahoo.com", + "failedTenant": "host4.yahoo.com", + "failedTenantResources": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "failedTenantAllocation": "allocated to tenant3.application3.instance3 as 'content/id3/0/0'", + "hostCandidateRejectionReasons": { + "singularReasonFailures": { + "insufficientVcpu": 0, + "insufficientMemoryGb": 0, + "insufficientDiskGb": 0, + "incompatibleDiskSpeed": 0, + "insufficientAvailableIps": 0, + "violatesParentHostPolicy": 1 + }, + "totalFailures": { + "insufficientVcpu": 0, + "insufficientMemoryGb": 0, + "insufficientDiskGb": 0, + "incompatibleDiskSpeed": 0, + "insufficientAvailableIps": 0, + "violatesParentHostPolicy": 1 + } + }, + "details": { + "hostsToRemove": [ + "dockerhost3.yahoo.com", + "dockerhost1.yahoo.com", + "dockerhost4.yahoo.com", + "dockerhost5.yahoo.com" + ], + "removalPossible": false, + "history": [ + { + "tenant": "test-node-pool-102-2", + "newParent": "dockerhost2.yahoo.com", + "eligibleParents": 1 + }, + { + "tenant": "host4.yahoo.com", + "eligibleParents": 0 + } + ] + } +} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json new file mode 100644 index 00000000000..d2d39b06161 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json @@ -0,0 +1,34 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/cfg1.yahoo.com", + "id": "cfg1.yahoo.com", + "state": "ready", + "type": "config", + "hostname": "cfg1.yahoo.com", + "openStackId": "cfg1", + "flavor": "default", + "cpuCores": 2.0, + "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.201.1", + "::201:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json new file mode 100644 index 00000000000..969c9a1b9c2 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json @@ -0,0 +1,34 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/cfg2.yahoo.com", + "id": "cfg2.yahoo.com", + "state": "ready", + "type": "config", + "hostname": "cfg2.yahoo.com", + "openStackId": "cfg2", + "flavor": "default", + "cpuCores": 2.0, + "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.202.1", + "::202:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/child-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/child-nodes.json new file mode 100644 index 00000000000..dae92aae091 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/child-nodes.json @@ -0,0 +1,5 @@ +{ + "nodes": [ + @include(node4.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json new file mode 100644 index 00000000000..e84f17dc40d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json @@ -0,0 +1,8 @@ +{ + "nodes": [ + @include(node6.json), + @include(docker-container1.json), + @include(node2.json), + @include(node4.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json new file mode 100644 index 00000000000..035e15ead49 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json @@ -0,0 +1,28 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/controller1.yahoo.com", + "id": "controller1.yahoo.com", + "state": "provisioned", + "type": "controller", + "hostname": "controller1.yahoo.com", + "openStackId": "fake-controller1.yahoo.com", + "flavor": "default", + "cpuCores": 2.0, + "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.0.1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json new file mode 100644 index 00000000000..c97509b3b3d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json @@ -0,0 +1,51 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/test-node-pool-102-2", + "id": "test-node-pool-102-2", + "state": "active", + "type": "tenant", + "hostname": "test-node-pool-102-2", + "parentHostname": "dockerhost3.yahoo.com", + "openStackId": "fake-test-node-pool-102-2", + "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote]", + "resources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" + }, + "membership": { + "clustertype": "content", + "clusterid": "id3", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0,"diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "reserved", + "at": 123, + "agent": "system" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::102:2" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json new file mode 100644 index 00000000000..77622933866 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json @@ -0,0 +1,73 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "cpuCores": 4.0, + "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed": "fast", "storageType":"remote" }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "currentOsVersion": "7.5.2", + "wantedOsVersion": "7.5.2", + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + }, + { + "event": "osUpgraded", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.100.1", + "::100:1" + ], + "additionalIpAddresses": [ + "::100:2", + "::100:3", + "::100:4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade.json new file mode 100644 index 00000000000..a1775dc794e --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade.json @@ -0,0 +1,68 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "cpuCores": 4.0, + "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast"}, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "currentOsVersion": "7.5.2", + "wantedOsVersion": "7.5.2", + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.100.1", + "::100:1" + ], + "additionalIpAddresses": [ + "::100:2", + "::100:3", + "::100:4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json new file mode 100644 index 00000000000..b6b93c8d108 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json @@ -0,0 +1,66 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "cpuCores": 4.0, + "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.100.1", + "::100:1" + ], + "additionalIpAddresses": [ + "::100:2", + "::100:3", + "::100:4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json new file mode 100644 index 00000000000..88e4d132ed5 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json @@ -0,0 +1,66 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", + "id": "dockerhost2.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost2.yahoo.com", + "openStackId": "dockerhost2", + "flavor": "large", + "cpuCores": 4.0, + "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.101.1", + "::101:1" + ], + "additionalIpAddresses": [ + "::101:2", + "::101:3", + "::101:4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json new file mode 100644 index 00000000000..c890f1e72be --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json @@ -0,0 +1,66 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com", + "id": "dockerhost3.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost3.yahoo.com", + "openStackId": "dockerhost3", + "flavor": "large", + "cpuCores": 4.0, + "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 2, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.102.1", + "::102:1" + ], + "additionalIpAddresses": [ + "::102:2", + "::102:3", + "::102:4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json new file mode 100644 index 00000000000..197a8c248e9 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json @@ -0,0 +1,66 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com", + "id": "dockerhost4.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost4.yahoo.com", + "openStackId": "dockerhost4", + "flavor": "large", + "cpuCores": 4.0, + "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 3, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.103.1", + "::103:1" + ], + "additionalIpAddresses": [ + "::103:2", + "::103:3", + "::103:4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json new file mode 100644 index 00000000000..2638b6b74f5 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json @@ -0,0 +1,66 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com", + "id": "dockerhost5.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost5.yahoo.com", + "openStackId": "dockerhost5", + "flavor": "large", + "cpuCores": 4.0, + "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 4, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.104.1", + "::104:1" + ], + "additionalIpAddresses": [ + "::104:2", + "::104:3", + "::104:4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json new file mode 100644 index 00000000000..ea5c3623303 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json @@ -0,0 +1,73 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "id": "dockerhost1.yahoo.com", + "state": "active", + "type": "host", + "hostname": "dockerhost1.yahoo.com", + "openStackId": "dockerhost1", + "flavor": "large", + "cpuCores": 4.0, + "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "owner": { + "tenant": "zoneapp", + "application": "zoneapp", + "instance": "zoneapp" + }, + "membership": { + "clustertype": "container", + "clusterid": "node-admin", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "currentFirmwareCheck": 100, + "wantedFirmwareCheck": 123, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + }, + { + "event": "firmwareVerified", + "at": 100, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.100.1", + "::100:1" + ], + "additionalIpAddresses": [ + "::100:2", + "::100:3", + "::100:4" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags1.json new file mode 100644 index 00000000000..a606777e9fd --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags1.json @@ -0,0 +1,10 @@ +{ + "flags": [ + { + "id": "exclusive-load-balancer", + "enabled": false, + "enabledHostnames": [], + "enabledApplications": ["zoneapp:zoneapp:zoneapp"] + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags2.json new file mode 100644 index 00000000000..4baf75f2169 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags2.json @@ -0,0 +1,15 @@ +{ + "flags": [ + { + "id": "exclusive-load-balancer", + "enabled": false, + "enabledHostnames": [ + "host1" + ], + "enabledApplications": [ + "zoneapp:zoneapp:zoneapp", + "foo:bar:default" + ] + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json new file mode 100644 index 00000000000..a9a728bab15 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json @@ -0,0 +1,34 @@ +{ + "loadBalancers": [ + { + "id": "tenant4:application4:instance4:id4", + "state": "active", + "changedAt": 123, + "application": "application4", + "tenant": "tenant4", + "instance": "instance4", + "cluster": "id4", + "hostname": "lb-tenant4.application4.instance4-id4", + "dnsZone": "zone-id-1", + "networks": [ + "10.2.3.0/24", + "10.4.5.0/24" + ], + "ports": [ + 4443 + ], + "reals": [ + { + "hostname": "host13.yahoo.com", + "ipAddress": "127.0.13.1", + "port": 4443 + }, + { + "hostname": "host14.yahoo.com", + "ipAddress": "127.0.14.1", + "port": 4443 + } + ] + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json new file mode 100644 index 00000000000..515081bcb8e --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json @@ -0,0 +1,64 @@ +{ + "loadBalancers": [ + { + "id": "tenant1:application1:instance1:id1", + "state": "reserved", + "changedAt": 123, + "application": "application1", + "tenant": "tenant1", + "instance": "instance1", + "cluster": "id1", + "hostname": "lb-tenant1.application1.instance1-id1", + "dnsZone": "zone-id-1", + "networks": [ + "10.2.3.0/24", + "10.4.5.0/24" + ], + "ports": [ + 4443 + ], + "reals": [ + { + "hostname": "host1.yahoo.com", + "ipAddress": "127.0.1.1", + "port": 4443 + }, + { + "hostname": "host10.yahoo.com", + "ipAddress": "127.0.10.1", + "port": 4443 + } + ] + }, + { + "id": "tenant4:application4:instance4:id4", + "state": "active", + "changedAt": 123, + "application": "application4", + "tenant": "tenant4", + "instance": "instance4", + "cluster": "id4", + "hostname": "lb-tenant4.application4.instance4-id4", + "dnsZone": "zone-id-1", + "networks": [ + "10.2.3.0/24", + "10.4.5.0/24" + ], + "ports": [ + 4443 + ], + "reals": [ + { + "hostname": "host13.yahoo.com", + "ipAddress": "127.0.13.1", + "port": 4443 + }, + { + "hostname": "host14.yahoo.com", + "ipAddress": "127.0.14.1", + "port": 4443 + } + ] + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json new file mode 100644 index 00000000000..e041a7b8b54 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json @@ -0,0 +1,64 @@ +{ + "jobs": [ + { + "name": "AutoscalingMaintainer" + }, + { + "name": "CapacityReportMaintainer" + }, + { + "name": "DirtyExpirer" + }, + { + "name": "FailedExpirer" + }, + { + "name": "InactiveExpirer" + }, + { + "name": "InfrastructureProvisioner" + }, + { + "name": "LoadBalancerExpirer" + }, + { + "name": "MetricsReporter" + }, + { + "name": "NodeFailer" + }, + { + "name": "NodeMetricsDbMaintainer" + }, + { + "name": "NodeRebooter" + }, + { + "name": "OperatorChangeApplicationMaintainer" + }, + { + "name": "OsUpgradeActivator" + }, + { + "name": "PeriodicApplicationMaintainer" + }, + { + "name": "ProvisionedExpirer" + }, + { + "name": "Rebalancer" + }, + { + "name": "ReservationExpirer" + }, + { + "name": "RetiredExpirer" + }, + { + "name":"ScalingSuggestionsMaintainer" + } + ], + "inactive": [ + "NodeFailer" + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json new file mode 100644 index 00000000000..7f6a6e17fa1 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json @@ -0,0 +1,56 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com", + "id": "host1.yahoo.com", + "state": "reserved", + "type": "tenant", + "hostname": "host1.yahoo.com", + "openStackId": "node1", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant1", + "application": "application1", + "instance": "instance1" + }, + "membership": { + "clustertype": "container", + "clusterid": "id1", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.1.1", + "::1:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json new file mode 100644 index 00000000000..da0e7b28f3e --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json @@ -0,0 +1,59 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com", + "id": "host10.yahoo.com", + "state": "reserved", + "type": "tenant", + "hostname": "host10.yahoo.com", + "parentHostname": "parent1.yahoo.com", + "openStackId": "node10", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant1", + "application": "application1", + "instance": "instance1" + }, + "membership": { + "clustertype": "container", + "clusterid": "id1", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "vespaVersion": "5.104.142", + "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:5.104.142", + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.10.1", + "::10:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json new file mode 100644 index 00000000000..17f8210fa4d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json @@ -0,0 +1,29 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host11.yahoo.com", + "id": "host11.yahoo.com", + "state": "provisioned", + "type": "tenant", + "hostname": "host11.yahoo.com", + "parentHostname": "parent.host.yahoo.com", + "openStackId": "host11.yahoo.com", + "flavor": "[vcpu: 1.0, memory: 1.0 Gb, disk 100.0 Gb, bandwidth: 0.3 Gbps]", + "resources":{"vcpu":1.0,"memoryGb":1.0,"diskGb":100.0,"bandwidthGbps":0.3,"diskSpeed":"fast","storageType":"any"}, + "environment": "DOCKER_CONTAINER", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "::11" + ], + "additionalIpAddresses": [ + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json new file mode 100644 index 00000000000..2e96c4c9bbc --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json @@ -0,0 +1,56 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host13.yahoo.com", + "id": "host13.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host13.yahoo.com", + "openStackId": "node13", + "flavor": "[vcpu: 10.0, memory: 48.0 Gb, disk 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":10.0,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant4", + "application": "application4", + "instance": "instance4" + }, + "membership": { + "clustertype": "container", + "clusterid": "id4", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.13.1", + "::13:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json new file mode 100644 index 00000000000..bfbecc09c60 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json @@ -0,0 +1,56 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host14.yahoo.com", + "id": "host14.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host14.yahoo.com", + "openStackId": "node14", + "flavor": "[vcpu: 10.0, memory: 48.0 Gb, disk 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":10.0,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0, "diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant4", + "application": "application4", + "instance": "instance4" + }, + "membership": { + "clustertype": "container", + "clusterid": "id4", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.14.1", + "::14:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json new file mode 100644 index 00000000000..b8ce5c86c4d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json @@ -0,0 +1,61 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com", + "id": "host2.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host2.yahoo.com", + "openStackId": "node2", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant2", + "application": "application2", + "instance": "instance2" + }, + "membership": { + "clustertype": "content", + "clusterid": "id2", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.2.1", + "::2:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json new file mode 100644 index 00000000000..70ae2a030bd --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json @@ -0,0 +1,33 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com", + "id": "host3.yahoo.com", + "state": "ready", + "type": "tenant", + "hostname": "host3.yahoo.com", + "openStackId": "node3", + "flavor": "[vcpu: 0.5, memory: 48.0 Gb, disk 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":0.5,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.3.1", + "::3:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json new file mode 100644 index 00000000000..af3552945d9 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json @@ -0,0 +1,81 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", + "id": "host4.yahoo.com", + "state": "failed", + "type": "tenant", + "hostname": "host4.yahoo.com", + "parentHostname": "parent.yahoo.com", + "openStackId": "patched-openstackid", + "flavor": "d-2-8-100", + "cpuCores": 2.0, + "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" + }, + "membership": { + "clustertype": "content", + "clusterid": "id3", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 1, + "currentRestartGeneration": 1, + "wantedDockerImage": "docker.domain.tld/my/image:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": true, + "suspendedSinceMillis": 0, + "rebootGeneration": 3, + "currentRebootGeneration": 1, + "vespaVersion": "6.43.0", + "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.45.0", + "failCount": 1, + "wantToRetire": true, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + }, + { + "event": "failed", + "at": 123, + "agent": "operator" + }, + { + "event": "rebooted", + "at": 123, + "agent": "system" + }, + { + "event": "wantToRetire", + "at": 123, + "agent": "operator" + } + ], + "ipAddresses": [ + "127.0.0.1", + "::1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json new file mode 100644 index 00000000000..c8680e1b420 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json @@ -0,0 +1,64 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", + "id": "host4.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host4.yahoo.com", + "parentHostname": "dockerhost1.yahoo.com", + "openStackId": "node4", + "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant3", + "application": "application3", + "instance": "instance3" + }, + "membership": { + "clustertype": "content", + "clusterid": "id3", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "vespaVersion": "6.41.0", + "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.41.0", + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.4.1", + "::4:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json new file mode 100644 index 00000000000..beb42ec6907 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json @@ -0,0 +1,39 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com", + "id": "host5.yahoo.com", + "state": "failed", + "type": "tenant", + "hostname": "host5.yahoo.com", + "parentHostname": "dockerhost2.yahoo.com", + "openStackId": "node5", + "flavor": "[vcpu: 1.0, memory: 8.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, disk speed: slow, storage type: remote]", + "resources":{"vcpu":1.0,"memoryGb":8.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"slow","storageType":"remote"}, + "environment": "DOCKER_CONTAINER", + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 1, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "failed", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.5.1", + "::5:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json new file mode 100644 index 00000000000..3fb5ffa4708 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json @@ -0,0 +1,41 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com", + "id": "host5.yahoo.com", + "state": "failed", + "type": "tenant", + "hostname": "host5.yahoo.com", + "parentHostname": "dockerhost2.yahoo.com", + "openStackId": "node5", + "flavor": "[vcpu: 1.0, memory: 8.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, disk speed: slow, storage type: remote]", + "resources":{"vcpu":1.0,"memoryGb":8.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"slow","storageType":"remote"}, + "environment": "DOCKER_CONTAINER", + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "vespaVersion": "1.2.3", + "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:1.2.3", + "failCount": 1, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "failed", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.5.1", + "::5:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json new file mode 100644 index 00000000000..3107311b792 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json @@ -0,0 +1,33 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com", + "id": "host55.yahoo.com", + "state": "dirty", + "type": "tenant", + "hostname": "host55.yahoo.com", + "openStackId": "node55", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": true, + "wantToDeprovision": true, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "deallocated", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.55.1", + "::55:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-after-changes.json new file mode 100644 index 00000000000..65c7e9db6cd --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-after-changes.json @@ -0,0 +1,60 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", + "id": "host6.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host6.yahoo.com", + "openStackId": "node6", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":1.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant2", + "application": "application2", + "instance": "instance2" + }, + "membership": { + "clustertype": "content", + "clusterid": "id2", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.6.1", + "::6:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json new file mode 100644 index 00000000000..a3d53798d7c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json @@ -0,0 +1,78 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", + "id": "host6.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host6.yahoo.com", + "openStackId": "node6", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant2", + "application": "application2", + "instance": "instance2" + }, + "membership": { + "clustertype": "content", + "clusterid": "id2", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": true, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.6.1", + "::6:1" + ], + "additionalIpAddresses": [], + "reports": { + "actualCpuCores": { + "createdMillis": 3 + }, + "diskSpace": { + "createdMillis": 2, + "description": "Actual disk space (2TB) differs from spec (3TB)", + "type":"HARD_FAIL", + "details": { + "inGib": 3, + "disks": [ + "/dev/sda1", + "/dev/sdb3" + ] + } + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json new file mode 100644 index 00000000000..7f0c3a5f706 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json @@ -0,0 +1,66 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", + "id": "host6.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host6.yahoo.com", + "openStackId": "node6", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant2", + "application": "application2", + "instance": "instance2" + }, + "membership": { + "clustertype": "content", + "clusterid": "id2", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.6.1", + "::6:1" + ], + "additionalIpAddresses": [], + "reports": { + "actualCpuCores": { + "createdMillis": 3 + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json new file mode 100644 index 00000000000..67b8d67c7f1 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json @@ -0,0 +1,81 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", + "id": "host6.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host6.yahoo.com", + "openStackId": "node6", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant2", + "application": "application2", + "instance": "instance2" + }, + "membership": { + "clustertype": "content", + "clusterid": "id2", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": true, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.6.1", + "::6:1" + ], + "additionalIpAddresses": [], + "reports": { + "actualCpuCores": { + "createdMillis": 1, + "description": "Actual number of CPU cores (2) differs from spec (4)", + "type":"HARD_FAIL", + "value": 2 + }, + "diskSpace": { + "createdMillis": 2, + "description": "Actual disk space (2TB) differs from spec (3TB)", + "type":"HARD_FAIL", + "details": { + "inGib": 3, + "disks": [ + "/dev/sda1", + "/dev/sdb3" + ] + } + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json new file mode 100644 index 00000000000..5a665a5a223 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json @@ -0,0 +1,61 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", + "id": "host6.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host6.yahoo.com", + "openStackId": "node6", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "owner": { + "tenant": "tenant2", + "application": "application2", + "instance": "instance2" + }, + "membership": { + "clustertype": "content", + "clusterid": "id2", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.6.1", + "::6:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json new file mode 100644 index 00000000000..38757b81156 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json @@ -0,0 +1,28 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com", + "id": "host7.yahoo.com", + "state": "provisioned", + "type": "tenant", + "hostname": "host7.yahoo.com", + "openStackId": "node7", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", + "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, + "environment": "DOCKER_CONTAINER", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.7.1", + "::7:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json new file mode 100644 index 00000000000..aa2b2acdb9f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json @@ -0,0 +1,28 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host8.yahoo.com", + "id": "host8.yahoo.com", + "state": "provisioned", + "type": "tenant", + "hostname": "host8.yahoo.com", + "openStackId": "host8.yahoo.com", + "flavor": "default", + "cpuCores": 2.0, + "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType": "remote"}, + "environment": "BARE_METAL", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.8.1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json new file mode 100644 index 00000000000..5f388c13f28 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json @@ -0,0 +1,29 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host9.yahoo.com", + "id": "host9.yahoo.com", + "state": "provisioned", + "type": "tenant", + "hostname": "host9.yahoo.com", + "openStackId": "host9.yahoo.com", + "flavor": "large-variant", + "cpuCores": 64.0, + "resources":{"vcpu":64.0,"memoryGb":128.0,"diskGb":2000.0,"bandwidthGbps":15.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.9.1", + "::9:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json new file mode 100644 index 00000000000..cf3e3c1b457 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json @@ -0,0 +1,23 @@ +{ + "nodes": [ + @include(node7.json), + @include(cfg1.json), + @include(node3.json), + @include(cfg2.json), + @include(node10.json), + @include(node1.json), + @include(docker-node4.json), + @include(node6.json), + @include(docker-container1.json), + @include(docker-node5.json), + @include(docker-node2.json), + @include(node13.json), + @include(node2.json), + @include(docker-node1.json), + @include(docker-node3.json), + @include(node14.json), + @include(node4.json), + @include(node55.json), + @include(node5.json) + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json new file mode 100644 index 00000000000..cdbd2cac9b9 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json @@ -0,0 +1,61 @@ +{ + "nodes": [ + { + "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/cfg1.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/cfg2.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/test-node-pool-102-2" + }, + { + "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host13.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host14.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com" + }, + { + "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com" + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent1.json new file mode 100644 index 00000000000..40f4a3b5160 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent1.json @@ -0,0 +1,33 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/parent1.yahoo.com", + "id": "parent1.yahoo.com", + "state": "ready", + "type": "host", + "hostname": "parent1.yahoo.com", + "openStackId": "parent1", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb]", + "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":1.0,"diskSpeed":"fast"}, + "environment": "BARE_METAL", + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.0.1", + "::1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json new file mode 100644 index 00000000000..ecc497172c7 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json @@ -0,0 +1,30 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/parent2.yahoo.com", + "id": "parent2.yahoo.com", + "state": "provisioned", + "type": "host", + "hostname": "parent2.yahoo.com", + "openStackId": "parent2.yahoo.com", + "flavor": "large-variant", + "reservedTo": "myTenant", + "cpuCores": 64.0, + "resources": {"vcpu":64.0,"memoryGb":128.0,"diskGb":2000.0,"bandwidthGbps":15.0,"diskSpeed":"fast","storageType":"remote"}, + "environment": "BARE_METAL", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.127.1", + "::127:1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/root.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/root.json new file mode 100644 index 00000000000..cc91314df84 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/root.json @@ -0,0 +1,22 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/nodes/v2/state/" + }, + { + "url": "http://localhost:8080/nodes/v2/node/" + }, + { + "url": "http://localhost:8080/nodes/v2/command/" + }, + { + "url": "http://localhost:8080/nodes/v2/maintenance/" + }, + { + "url":"http://localhost:8080/nodes/v2/upgrade/" + }, + { + "url":"http://localhost:8080/nodes/v2/application/" + } + ] +} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json new file mode 100644 index 00000000000..f8343756559 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json @@ -0,0 +1,68 @@ +{ + "states": { + "provisioned": { + "url": "http://localhost:8080/nodes/v2/state/provisioned", + "nodes": [ + @include(node7.json) + ] + }, + "ready": { + "url": "http://localhost:8080/nodes/v2/state/ready", + "nodes": [ + @include(node3.json), + @include(cfg1.json), + @include(cfg2.json) + ] + }, + "reserved": { + "url": "http://localhost:8080/nodes/v2/state/reserved", + "nodes": [ + @include(node10.json), + @include(node1.json) + ] + }, + "active": { + "url": "http://localhost:8080/nodes/v2/state/active", + "nodes": [ + @include(node6.json), + @include(docker-container1.json), + @include(node13.json), + @include(node2.json), + @include(node14.json), + @include(node4.json), + @include(docker-node4.json), + @include(docker-node5.json), + @include(docker-node2.json), + @include(docker-node1.json), + @include(docker-node3.json) + ] + }, + "inactive": { + "url": "http://localhost:8080/nodes/v2/state/inactive", + "nodes": [ + ] + }, + "dirty": { + "url": "http://localhost:8080/nodes/v2/state/dirty", + "nodes": [ + @include(node55.json) + ] + }, + "failed": { + "url": "http://localhost:8080/nodes/v2/state/failed", + "nodes": [ + @include(node5.json) + ] + }, + "parked": { + "url": "http://localhost:8080/nodes/v2/state/parked", + "nodes": [ + ] + }, + "deprovisioned": { + "url": "http://localhost:8080/nodes/v2/state/deprovisioned", + "nodes": [ + ] + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states.json new file mode 100644 index 00000000000..69579148df3 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states.json @@ -0,0 +1,31 @@ +{ + "states": { + "provisioned": { + "url": "http://localhost:8080/nodes/v2/state/provisioned" + }, + "ready": { + "url": "http://localhost:8080/nodes/v2/state/ready" + }, + "reserved": { + "url": "http://localhost:8080/nodes/v2/state/reserved" + }, + "active": { + "url": "http://localhost:8080/nodes/v2/state/active" + }, + "inactive": { + "url": "http://localhost:8080/nodes/v2/state/inactive" + }, + "dirty": { + "url": "http://localhost:8080/nodes/v2/state/dirty" + }, + "failed": { + "url": "http://localhost:8080/nodes/v2/state/failed" + }, + "parked": { + "url": "http://localhost:8080/nodes/v2/state/parked" + }, + "deprovisioned": { + "url": "http://localhost:8080/nodes/v2/state/deprovisioned" + } + } +} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java deleted file mode 100644 index ed2aae64b5f..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author mpolden - */ -public class NodeSerializerTest { - - @Test - public void serialize_node_types() { - for (NodeType t : NodeType.values()) { - assertEquals(t, NodeSerializer.typeFrom(NodeSerializer.toString(t))); - } - } - - @Test - public void serialize_node_states() { - for (Node.State s : Node.State.values()) { - assertEquals(s, NodeSerializer.stateFrom(NodeSerializer.toString(s))); - } - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java deleted file mode 100644 index 838a39ba01e..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ /dev/null @@ -1,1044 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.application.Networking; -import com.yahoo.application.container.JDisc; -import com.yahoo.application.container.handler.Request; -import com.yahoo.application.container.handler.Response; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.maintenance.OsUpgradeActivator; -import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; -import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; -import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; -import org.junit.After; -import org.junit.Before; -import org.junit.ComparisonFailure; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -/** - * Test of the REST APIs provided by the node repository. - * - * Note: This class is referenced from our operations documentation and must not be renamed/moved without updating that. - * - * @author bratseth - */ -public class RestApiTest { - - private final static String responsesPath = "src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/"; - - private JDisc container; - - @Before - public void startContainer() { - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); - } - - @After - public void stopContainer() { - if (container != null) container.close(); - } - - /** This test gives examples of the node requests that can be made to nodes/v2 */ - @Test - public void test_requests() throws Exception { - // GET - assertFile(new Request("http://localhost:8080/nodes/v2/"), "root.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/state/"), "states.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/state/?recursive=true"), "states-recursive.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/state/active?recursive=true"), "active-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/"), "nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true"), "nodes-recursive.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "node2.json"); - - // GET with filters - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host6.yahoo.com%20host2.yahoo.com"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "content-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterId=id2"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&application=tenant2.application2.instance2"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=dockerhost1.yahoo.com"), "child-nodes.json"); - - // POST restart command - assertRestart(1, new Request("http://localhost:8080/nodes/v2/command/restart?hostname=host2.yahoo.com", - new byte[0], Request.Method.POST)); - assertRestart(2, new Request("http://localhost:8080/nodes/v2/command/restart?application=tenant2.application2.instance2", - new byte[0], Request.Method.POST)); - assertRestart(11, new Request("http://localhost:8080/nodes/v2/command/restart", - new byte[0], Request.Method.POST)); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), - "\"restartGeneration\":3"); - - // POST reboot command - assertReboot(12, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", - new byte[0], Request.Method.POST)); - assertReboot(2, new Request("http://localhost:8080/nodes/v2/command/reboot?application=tenant2.application2.instance2", - new byte[0], Request.Method.POST)); - assertReboot(19, new Request("http://localhost:8080/nodes/v2/command/reboot", - new byte[0], Request.Method.POST)); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), - "\"rebootGeneration\":4"); - - // POST new nodes - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.8.1") + "," + // test with only 1 ip address - asNodeJson("host9.yahoo.com", "large-variant", "127.0.9.1", "::9:1") + "," + - asHostJson("parent2.yahoo.com", "large-variant", Optional.of(TenantName.from("myTenant")), "127.0.127.1", "::127:1") + "," + - asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com", "::11") + "]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - "{\"message\":\"Added 4 nodes to the provisioned state\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), "node8.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com"), "node9.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com"), "node11.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/parent2.yahoo.com"), "parent2.json"); - - // POST duplicate node - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.8") + "]").getBytes(StandardCharsets.UTF_8), - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); - - // DELETE a provisioned node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com", - new byte[0], Request.Method.DELETE), - "{\"message\":\"Removed host9.yahoo.com\"}"); - - // PUT nodes ready - assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host8.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host8.yahoo.com to dirty\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host8.yahoo.com to ready\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), - "\"state\":\"ready\""); - // calling ready again is a noop: - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host8.yahoo.com to ready\"}"); - - // PUT a node in failed ... - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host2.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host2.yahoo.com to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), - "\"state\":\"failed\""); - // ... and put it back in active (after fixing). This is useful to restore data when multiple nodes fail. - assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host2.yahoo.com to active\"}"); - - // PUT a node in parked ... - assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host8.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host8.yahoo.com to parked\"}"); - assertResponseContains(new Request("http://localhost:8080()/nodes/v2/node/host8.yahoo.com"), - "\"state\":\"parked\""); - // ... and delete it - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com", - new byte[0], Request.Method.DELETE), - "{\"message\":\"Removed host8.yahoo.com\"}"); - - // or, PUT a node in failed ... - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/test-node-pool-102-2", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved test-node-pool-102-2 to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), - "\"state\":\"failed\""); - // ... and deallocate it such that it moves to dirty and is recycled - assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/test-node-pool-102-2", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved test-node-pool-102-2 to dirty\"}"); - - // ... and set it back to ready as if this was from the node-admin with the temporary state rest api - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/test-node-pool-102-2", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved test-node-pool-102-2 to ready\"}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", new byte[0], Request.Method.GET), - 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'test-node-pool-102-2'\"}"); - - // Put a host in failed and make sure it's children are also failed - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/dockerhost1.yahoo.com", new byte[0], Request.Method.PUT), - "{\"message\":\"Moved dockerhost1.yahoo.com, host4.yahoo.com to failed\"}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed"), "{\"nodes\":[" + - "{\"url\":\"http://localhost:8080/nodes/v2/node/host5.yahoo.com\"}," + - "{\"url\":\"http://localhost:8080/nodes/v2/node/host4.yahoo.com\"}," + - "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}]}"); - - // Update (PATCH) a node (multiple fields can also be sent in one request body) - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"currentRebootGeneration\": 1}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"flavor\": \"d-2-8-100\"}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"currentVespaVersion\": \"5.104.142\"}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"parentHostname\": \"parent.yahoo.com\"}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"ipAddresses\": [\"127.0.0.1\",\"::1\"]}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"wantToRetire\": true}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"currentVespaVersion\": \"6.43.0\",\"currentDockerImage\": \"docker-registry.domain.tld:8080/dist/vespa:6.45.0\"}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"openStackId\": \"patched-openstackid\"}"), Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), - "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), - "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - Utf8.toBytes("{\"modelName\": null}"), Request.Method.PATCH), - "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "modelName", false); - container.handleRequest((new Request("http://localhost:8080/nodes/v2/upgrade/tenant", Utf8.toBytes("{\"dockerImage\": \"docker.domain.tld/my/image\"}"), Request.Method.PATCH))); - - ((OrchestratorMock) container.components().getComponent(OrchestratorMock.class.getName())) - .suspend(new HostName("host4.yahoo.com")); - - assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); - } - - @Test - public void test_application_requests() throws Exception { - assertFile(new Request("http://localhost:8080/nodes/v2/application/"), "applications.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/application/tenant1.application1.instance1"), - "application1.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/application/tenant2.application2.instance2"), - "application2.json"); - } - - @Test - public void maintenance_requests() throws Exception { - // POST deactivation of a maintenance job - assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/inactive/NodeFailer", - new byte[0], Request.Method.POST), - "{\"message\":\"Deactivated job 'NodeFailer'\"}"); - // GET a list of all maintenance jobs - assertFile(new Request("http://localhost:8080/nodes/v2/maintenance/"), "maintenance.json"); - - // DELETE deactivation of a maintenance job - assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/inactive/NodeFailer", - new byte[0], Request.Method.DELETE), - "{\"message\":\"Re-activated job 'NodeFailer'\"}"); - - // POST run of a maintenance job - assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/run/PeriodicApplicationMaintainer", - new byte[0], Request.Method.POST), - "{\"message\":\"Executed job 'PeriodicApplicationMaintainer'\"}"); - - // POST run of unknown maintenance job - assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/run/foo", - new byte[0], Request.Method.POST), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such job 'foo'\"}"); - } - - @Test - public void post_with_patch_method_override_in_header_is_handled_as_patch() throws Exception { - Request req = new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.POST); - req.getHeaders().add("X-HTTP-Method-Override", "PATCH"); - assertResponse(req, "{\"message\":\"Updated host4.yahoo.com\"}"); - } - - @Test - public void post_with_invalid_method_override_in_header_gives_sane_error_message() throws Exception { - Request req = new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.POST); - req.getHeaders().add("X-HTTP-Method-Override", "GET"); - assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Illegal X-HTTP-Method-Override header for POST request. Accepts 'PATCH' but got 'GET'\"}"); - } - - @Test - public void post_node_with_ip_address() throws Exception { - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("ipv4-host.yahoo.com", "default","127.0.0.1") + "]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("ipv6-host.yahoo.com", "default", "::1") + "]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("dual-stack-host.yahoo.com", "default", "127.0.254.254", "::254:254") + "]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - } - - @Test - public void post_node_with_duplicate_ip_address() throws Exception { - Request req = new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("host-with-ip.yahoo.com", "default", "foo") + "]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST); - assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Found one or more invalid addresses in [foo]: 'foo' is not an IP string literal.\"}"); - - // Attempt to POST tenant node with already assigned IP - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("tenant-node-foo.yahoo.com", "default", "127.0.1.1") + "]", - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.1.1] to tenant-node-foo.yahoo.com: [127.0.1.1] already assigned to host1.yahoo.com\"}"); - - // Attempt to PATCH existing tenant node with already assigned IP - assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", - "{\"ipAddresses\": [\"127.0.2.1\"]}", - Request.Method.PATCH), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [127.0.2.1] to test-node-pool-102-2: [127.0.2.1] already assigned to host2.yahoo.com\"}"); - - // Attempt to POST host node with already assigned IP - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asHostJson("host200.yahoo.com", "default", Optional.empty(), "127.0.2.1") + "]", - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.2.1] to host200.yahoo.com: [127.0.2.1] already assigned to host2.yahoo.com\"}"); - - // Attempt to PATCH host node with IP in the pool of another node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - "{\"ipAddresses\": [\"::104:3\"]}", - Request.Method.PATCH), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [::100:4, ::100:3, ::100:2, ::104:3] to dockerhost1.yahoo.com: [::104:3] already assigned to dockerhost5.yahoo.com\"}"); - - // Node types running a single container can share their IP address with child node - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.42.1") + "]", - Request.Method.POST), 200, - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asDockerNodeJson("cfg42.yahoo.com", NodeType.config, "cfghost42.yahoo.com", "127.0.42.1") + "]", - Request.Method.POST), 200, - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - - // ... but cannot share with child node of wrong type - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asDockerNodeJson("proxy42.yahoo.com", NodeType.proxy, "cfghost42.yahoo.com", "127.0.42.1") + "]", - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.42.1] to proxy42.yahoo.com: [127.0.42.1] already assigned to cfg42.yahoo.com\"}"); - - // ... nor with child node on different host - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.43.1") + "]", - Request.Method.POST), 200, - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", - "{\"ipAddresses\": [\"127.0.43.1\"]}", - Request.Method.PATCH), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [127.0.43.1] to cfg42.yahoo.com: [127.0.43.1] already assigned to cfghost43.yahoo.com\"}"); - } - - @Test - public void post_controller_node() throws Exception { - String data = "[{\"hostname\":\"controller1.yahoo.com\", \"openStackId\":\"fake-controller1.yahoo.com\"," + - createIpAddresses("127.0.0.1") + - "\"flavor\":\"default\"" + - ", \"type\":\"controller\"}]"; - assertResponse(new Request("http://localhost:8080/nodes/v2/node", data.getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - - assertFile(new Request("http://localhost:8080/nodes/v2/node/controller1.yahoo.com"), "controller1.json"); - } - - @Test - public void fails_to_ready_node_with_hard_fail() throws Exception { - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("host12.yahoo.com", "default") + "]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - String msg = "Actual disk space (2TB) differs from spec (3TB)"; - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host12.yahoo.com", - Utf8.toBytes("{\"reports\":{\"diskSpace\":{\"createdMillis\":2,\"description\":\"" + msg + "\",\"type\": \"HARD_FAIL\"}}}"), - Request.Method.PATCH), - "{\"message\":\"Updated host12.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host12.yahoo.com", new byte[0], Request.Method.PUT), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"provisioned host host12.yahoo.com cannot be readied because it has " + - "hard failures: [diskSpace reported 1970-01-01T00:00:00.002Z: " + msg + "]\"}"); - } - - @Test - public void patching_dirty_node_does_not_increase_reboot_generation() throws Exception { - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("foo.yahoo.com", "default") + "]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/foo.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved foo.yahoo.com to failed\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/foo.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved foo.yahoo.com to dirty\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), - "\"rebootGeneration\":1"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com", - Utf8.toBytes("{\"currentRebootGeneration\": 42}"), Request.Method.PATCH), - "{\"message\":\"Updated foo.yahoo.com\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), - "\"rebootGeneration\":1"); - } - - @Test - public void acl_request_by_tenant_node() throws Exception { - String hostname = "foo.yahoo.com"; - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson(hostname, "default", "127.0.222.1") + "]").getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/" + hostname, - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved foo.yahoo.com to dirty\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/" + hostname, - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved foo.yahoo.com to ready\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/acl/" + hostname), "acl-tenant-node.json"); - } - - @Test - public void acl_request_by_config_server() throws Exception { - assertFile(new Request("http://localhost:8080/nodes/v2/acl/cfg1.yahoo.com"), "acl-config-server.json"); - } - - @Test - public void test_invalid_requests() throws Exception { - assertResponse(new Request("http://localhost:8080/nodes/v2/node/node-does-not-exist", - new byte[0], Request.Method.GET), - 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'node-does-not-exist'\"}"); - - // Attempt to fail and ready an allocated node without going through dirty - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/node-does-not-exist", - new byte[0], Request.Method.PUT), - 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not move node-does-not-exist to failed: Node not found\"}"); - - // Attempt to fail and ready an allocated node without going through dirty - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host1.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host1.yahoo.com to failed\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com", - new byte[0], Request.Method.PUT), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make failed host host1.yahoo.com allocated to tenant1.application1.instance1 as 'container/id1/0/0' available for new allocation as it is not in state [dirty]\"}"); - - // (... while dirty then ready works (the ready move will be initiated by node maintenance)) - assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host1.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host1.yahoo.com to dirty\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host1.yahoo.com to ready\"}"); - - // Attempt to park and ready an allocated node without going through dirty - assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host2.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host2.yahoo.com to parked\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com", - new byte[0], Request.Method.PUT), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make parked host host2.yahoo.com allocated to tenant2.application2.instance2 as 'content/id2/0/0' available for new allocation as it is not in state [dirty]\"}"); - // (... while dirty then ready works (the ready move will be initiated by node maintenance)) - assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host2.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host2.yahoo.com to dirty\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com", - new byte[0], Request.Method.PUT), - "{\"message\":\"Moved host2.yahoo.com to ready\"}"); - - // Attempt to DELETE a node which has been removed - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com", - new byte[0], Request.Method.DELETE), - 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'host2.yahoo.com'\"}"); - - // Attempt to DELETE allocated node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - new byte[0], Request.Method.DELETE), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"active child node host4.yahoo.com allocated to tenant3.application3.instance3 as 'content/id3/0/0' is currently allocated and cannot be removed\"}"); - - // PUT current restart generation with string instead of long - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": \"1\"}"), Request.Method.PATCH), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Expected a LONG value, got a STRING\"}"); - - // PUT flavor with long instead of string - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"flavor\": 1}"), Request.Method.PATCH), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'flavor': Expected a STRING value, got a LONG\"}"); - - // Attempt to set nonexisting node to active - assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", - new byte[0], Request.Method.PUT), 404, - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not move host2.yahoo.com to active: Node not found\"}"); - - // Attempt to POST duplicate nodes - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.1", "::254:1") + "," + - asNodeJson("host8.yahoo.com", "large-variant", "127.0.253.1", "::253:1") + "]").getBytes(StandardCharsets.UTF_8), - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add nodes: provisioned host host8.yahoo.com is duplicated in the argument list\"}"); - - // Attempt to PATCH field not relevant for child node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", - Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'modelName': A child node cannot have model name set\"}"); - } - - @Test - public void test_node_patching() throws Exception { - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{" + - "\"currentRestartGeneration\": 1," + - "\"currentRebootGeneration\": 3," + - "\"flavor\": \"medium-disk\"," + - "\"currentVespaVersion\": \"5.104.142\"," + - "\"failCount\": 0," + - "\"parentHostname\": \"parent.yahoo.com\"" + - "}" - ), - Request.Method.PATCH), - "{\"message\":\"Updated host4.yahoo.com\"}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/node/doesnotexist.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": 1}"), - Request.Method.PATCH), - 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node found with hostname doesnotexist.yahoo.com\"}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": 1}"), - Request.Method.PATCH), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Node is not allocated\"}"); - } - - @Test - public void test_node_patch_to_remove_docker_ready_fields() throws Exception { - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com", - Utf8.toBytes("{" + - "\"currentVespaVersion\": \"\"," + - "\"currentDockerImage\": \"\"" + - "}" - ), - Request.Method.PATCH), - "{\"message\":\"Updated host5.yahoo.com\"}"); - - assertFile(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com"), "node5-after-changes.json"); - } - - @Test - public void test_reports_patching() throws IOException { - // Add report - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", - Utf8.toBytes("{" + - " \"reports\": {" + - " \"actualCpuCores\": {" + - " \"createdMillis\": 1, " + - " \"description\": \"Actual number of CPU cores (2) differs from spec (4)\"," + - " \"type\": \"HARD_FAIL\"," + - " \"value\":2" + - " }," + - " \"diskSpace\": {" + - " \"createdMillis\": 2, " + - " \"description\": \"Actual disk space (2TB) differs from spec (3TB)\"," + - " \"type\": \"HARD_FAIL\"," + - " \"details\": {" + - " \"inGib\": 3," + - " \"disks\": [\"/dev/sda1\", \"/dev/sdb3\"]" + - " }" + - " }" + - " }" + - "}"), - Request.Method.PATCH), - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); - - // Patching with an empty reports is no-op - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", - Utf8.toBytes("{\"reports\": {}}"), - Request.Method.PATCH), - 200, - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); - - // Patching existing report overwrites - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", - Utf8.toBytes("{" + - " \"reports\": {" + - " \"actualCpuCores\": {" + - " \"createdMillis\": 3 " + - " }" + - " }" + - "}"), - Request.Method.PATCH), - 200, - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports-2.json"); - - // Clearing one report - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", - Utf8.toBytes("{\"reports\": { \"diskSpace\": null } }"), - Request.Method.PATCH), - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports-3.json"); - - // Clearing all reports - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", - Utf8.toBytes("{\"reports\": null }"), - Request.Method.PATCH), - "{\"message\":\"Updated host6.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6.json"); - } - - @Test - public void test_upgrade() throws IOException { - // Initially, no versions are set - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), "{\"versions\":{},\"osVersions\":{},\"dockerImages\":{}}"); - - // Set version for config, confighost and controller - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", - Utf8.toBytes("{\"version\": \"6.123.456\"}"), - Request.Method.PATCH), - "{\"message\":\"Set version to 6.123.456 for nodes of type config\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"version\": \"6.123.456\"}"), - Request.Method.PATCH), - "{\"message\":\"Set version to 6.123.456 for nodes of type confighost\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/controller", - Utf8.toBytes("{\"version\": \"6.123.456\"}"), - Request.Method.PATCH), - "{\"message\":\"Set version to 6.123.456 for nodes of type controller\"}"); - - - // Verify versions are set - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); - - // Setting version for unsupported node type fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/tenant", - Utf8.toBytes("{\"version\": \"6.123.456\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Target version for type tenant is not allowed\"}"); - - // Omitting version field fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"At least one of 'version', 'osVersion' or 'dockerImage' must be set\"}"); - - // Downgrade without force fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"version\": \"6.123.1\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. " + - "Current target version: 6.123.456, attempted to set target version: 6.123.1\"}"); - - // Downgrade with force is OK - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"version\": \"6.123.1\",\"force\":true}"), - Request.Method.PATCH), - "{\"message\":\"Set version to 6.123.1 for nodes of type confighost\"}"); - - // Verify version has been updated - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); - - // Setting empty version without force fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"version\": null}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. Current target version: 6.123.1, attempted to set target version: 0.0.0\"}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"version\": null, \"force\": true}"), - Request.Method.PATCH), - "{\"message\":\"Set version to 0.0.0 for nodes of type confighost\"}"); - - // Verify version has been removed - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); - - // Upgrade OS for confighost and host - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), - Request.Method.PATCH), - "{\"message\":\"Set osVersion to 7.5.2 for nodes of type confighost\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host", - Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), - Request.Method.PATCH), - "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}"); - - // OS versions are set - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"controller\":\"6.123.456\"},\"osVersions\":{\"host\":\"7.5.2\",\"confighost\":\"7.5.2\"},\"dockerImages\":{}}"); - - // Upgrade OS and Vespa together - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"version\": \"6.124.42\", \"osVersion\": \"7.5.2\"}"), - Request.Method.PATCH), - "{\"message\":\"Set version to 6.124.42, osVersion to 7.5.2 for nodes of type confighost\"}"); - - // Attempt to upgrade unsupported node type - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", - Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Node type 'config' does not support OS upgrades\"}"); - - // Attempt to downgrade OS - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"osVersion\": \"7.4.2\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set target OS version to 7.4.2 without setting 'force', as it's lower than the current version: 7.5.2\"}"); - - // Downgrading OS with force succeeds - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"osVersion\": \"7.4.2\", \"force\": true}"), - Request.Method.PATCH), - "{\"message\":\"Set osVersion to 7.4.2 for nodes of type confighost\"}"); - - // Current target is considered bad, remove it - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"osVersion\": null}"), - Request.Method.PATCH), - 200, - "{\"message\":\"Set osVersion to null for nodes of type confighost\"}"); - - // Set docker image for config and tenant - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/tenant", - Utf8.toBytes("{\"dockerImage\": \"my-repo.my-domain.example:1234/repo/tenant\"}"), - Request.Method.PATCH), - "{\"message\":\"Set docker image to my-repo.my-domain.example:1234/repo/tenant for nodes of type tenant\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", - Utf8.toBytes("{\"dockerImage\": \"my-repo.my-domain.example:1234/repo/image\"}"), - Request.Method.PATCH), - "{\"message\":\"Set docker image to my-repo.my-domain.example:1234/repo/image for nodes of type config\"}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/"), - "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.124.42\",\"controller\":\"6.123.456\"},\"osVersions\":{\"host\":\"7.5.2\"},\"dockerImages\":{\"tenant\":\"my-repo.my-domain.example:1234/repo/tenant\",\"config\":\"my-repo.my-domain.example:1234/repo/image\"}}"); - - // Cannot set docker image for non docker node type - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"dockerImage\": \"my-repo.my-domain.example:1234/repo/image\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Setting docker image for confighost nodes is unsupported\"}"); - } - - @Test - public void test_os_version() throws Exception { - // Schedule OS upgrade - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/host", - Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), - Request.Method.PATCH), - "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}"); - - // Activate target - var nodeRepository = (NodeRepository) container.components().getComponent(MockNodeRepository.class.getName()); - var osUpgradeActivator = new OsUpgradeActivator(nodeRepository, Duration.ofDays(1)); - osUpgradeActivator.run(); - - // Other node type does not return wanted OS version - Response r = container.handleRequest(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com")); - assertFalse("Response omits wantedOsVersions field", r.getBodyAsString().contains("wantedOsVersion")); - - // Node updates its node object after upgrading OS - assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"), - Request.Method.PATCH), - "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-os-upgrade-complete.json"); - - // Another node upgrades - assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", - Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"), - Request.Method.PATCH), - "{\"message\":\"Updated dockerhost2.yahoo.com\"}"); - - // Filter nodes by osVersion - assertResponse(new Request("http://localhost:8080/nodes/v2/node/?osVersion=7.5.2"), - "{\"nodes\":[" + - "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com\"}," + - "{\"url\":\"http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com\"}" + - "]}"); - } - - @Test - public void test_firmware_upgrades() throws IOException { - // dockerhost1 checks firmware at time 100. - assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - Utf8.toBytes("{\"currentFirmwareCheck\":100}"), Request.Method.PATCH), - "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - - // Schedule a firmware check at time 123 (the mock default). - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/firmware", new byte[0], Request.Method.POST), - "{\"message\":\"Will request firmware checks on all hosts.\"}"); - - // dockerhost1 displays both values. - assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), - "dockerhost1-with-firmware-data.json"); - - // host1 has no wantedFirmwareCheck, as it's not a docker host. - assertFile(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), - "node1.json"); - - // Cancel the firmware check. - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/firmware", new byte[0], Request.Method.DELETE), - "{\"message\":\"Cancelled outstanding requests for firmware checks\"}"); - } - - @Test - public void test_capacity() throws Exception { - assertFile(new Request("http://localhost:8080/nodes/v2/capacity/?json=true"), "capacity-zone.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/capacity?json=true"), "capacity-zone.json"); - - List hostsToRemove = List.of( - "dockerhost1.yahoo.com", - "dockerhost2.yahoo.com", - "dockerhost3.yahoo.com", - "dockerhost4.yahoo.com" - ); - String requestUriTemplate = "http://localhost:8080/nodes/v2/capacity/?json=true&hosts=%s"; - - assertFile(new Request(String.format(requestUriTemplate, - String.join(",", hostsToRemove.subList(0, 3)))), - "capacity-hostremoval-possible.json"); - assertFile(new Request(String.format(requestUriTemplate, - String.join(",", hostsToRemove))), - "capacity-hostremoval-impossible.json"); - } - - - /** Tests the rendering of each node separately to make it easier to find errors */ - @Test - public void test_single_node_rendering() throws Exception { - for (int i = 1; i <= 14; i++) { - if (i == 8 || i == 9 || i == 11 || i == 12) continue; // these nodes are added later - assertFile(new Request("http://localhost:8080/nodes/v2/node/host" + i + ".yahoo.com"), "node" + i + ".json"); - } - } - - @Test - public void test_load_balancers() throws Exception { - assertFile(new Request("http://localhost:8080/loadbalancers/v1/"), "load-balancers.json"); - assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant4.application4.instance4"), "load-balancers-single.json"); - assertResponse(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant.nonexistent.default"), "{\"loadBalancers\":[]}"); - } - - @Test - public void test_flavor_overrides() throws Exception { - String host = "parent2.yahoo.com"; - // Test adding with overrides - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[{\"hostname\":\"" + host + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + - "\"flavor\":\"large-variant\",\"resources\":{\"diskGb\":1234,\"memoryGb\":4321}}]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can only override disk GB for configured flavor\"}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[{\"hostname\":\"" + host + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + - "\"flavor\":\"large-variant\",\"type\":\"host\",\"resources\":{\"diskGb\":1234}}]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), - "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":1234.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); - - // Test adding tenant node - String tenant = "node-1-3.yahoo.com"; - String resources = "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":1234.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"slow\",\"storageType\":\"remote\"}"; - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[{\"hostname\":\"" + tenant + "\"," + createIpAddresses("::2") + "\"openStackId\":\"osid-124\"," + - "\"type\":\"tenant\"," + resources + "}]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + tenant), resources); - - // Test patching with overrides - assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, - "{\"minDiskAvailableGb\":5432,\"minMainMemoryAvailableGb\":2345}".getBytes(StandardCharsets.UTF_8), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'minMainMemoryAvailableGb': Can only override disk GB for configured flavor\"}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, - "{\"minDiskAvailableGb\":5432}".getBytes(StandardCharsets.UTF_8), - Request.Method.PATCH), - "{\"message\":\"Updated " + host + "\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), - "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":5432.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); - } - - @Test - public void test_node_resources() throws Exception { - String hostname = "node123.yahoo.com"; - String resources = "\"resources\":{\"vcpu\":5.0,\"memoryGb\":4321.0,\"diskGb\":1234.0,\"bandwidthGbps\":0.3,\"diskSpeed\":\"slow\",\"storageType\":\"local\"}"; - // Test adding new node with resources - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[{\"hostname\":\"" + hostname + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + - resources.replace("\"memoryGb\":4321.0,", "") + "}]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Required field 'memoryGb' is missing\"}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[{\"hostname\":\"" + hostname + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + resources + "}]") - .getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), resources); - - // Test patching with overrides - assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname, - "{\"diskGb\":12,\"memoryGb\":34,\"vcpu\":56,\"fastDisk\":true,\"remoteStorage\":true,\"bandwidthGbps\":78.0}".getBytes(StandardCharsets.UTF_8), - Request.Method.PATCH), - "{\"message\":\"Updated " + hostname + "\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), - "\"resources\":{\"vcpu\":56.0,\"memoryGb\":34.0,\"diskGb\":12.0,\"bandwidthGbps\":78.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); - } - - private static String asDockerNodeJson(String hostname, String parentHostname, String... ipAddress) { - return asDockerNodeJson(hostname, NodeType.tenant, parentHostname, ipAddress); - } - - private static String asDockerNodeJson(String hostname, NodeType nodeType, String parentHostname, String... ipAddress) { - return "{\"hostname\":\"" + hostname + "\", \"parentHostname\":\"" + parentHostname + "\"," + - createIpAddresses(ipAddress) + - "\"openStackId\":\"" + hostname + "\",\"flavor\":\"d-1-1-100\"" + - (nodeType != NodeType.tenant ? ",\"type\":\"" + nodeType + "\"" : "") + - "}"; - } - - private static String asNodeJson(String hostname, String flavor, String... ipAddress) { - return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\"," + - createIpAddresses(ipAddress) + - "\"flavor\":\"" + flavor + "\"}"; - } - - private static String asHostJson(String hostname, String flavor, Optional reservedTo, String... ipAddress) { - return asNodeJson(hostname, NodeType.host, flavor, reservedTo, ipAddress); - } - - private static String asNodeJson(String hostname, NodeType nodeType, String flavor, Optional reservedTo, String... ipAddress) { - return "{\"hostname\":\"" + hostname + "\", \"openStackId\":\"" + hostname + "\"," + - createIpAddresses(ipAddress) + - "\"flavor\":\"" + flavor + "\"" + - (reservedTo.isPresent() ? ", \"reservedTo\":\"" + reservedTo.get().value() + "\"" : "") + - ", \"type\":\"" + nodeType + "\"}"; - } - - private static String createIpAddresses(String... ipAddress) { - return "\"ipAddresses\":[" + - Arrays.stream(ipAddress) - .map(ip -> "\"" + ip + "\"") - .collect(Collectors.joining(",")) + - "],"; - } - - /** Asserts a particular response and 200 as response status */ - private void assertResponse(Request request, String responseMessage) throws IOException { - assertResponse(request, 200, responseMessage); - } - - private void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException { - Response response = container.handleRequest(request); - // Compare both status and message at once for easier diagnosis - assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage, - "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString()); - } - - private void assertResponseContains(Request request, String responseSnippet) throws IOException { - assertPartialResponse(request, responseSnippet, true); - } - - private void assertPartialResponse(Request request, String responseSnippet, boolean match) throws IOException { - String response = container.handleRequest(request).getBodyAsString(); - assertEquals(String.format("Expected response to " + (match ? " " : "not ") + "contain: %s\nResponse: %s", - responseSnippet, response), match, response.contains(responseSnippet)); - } - - private void assertFile(Request request, String responseFile) throws IOException { - String expectedResponse = IOUtils.readFile(new File(responsesPath + responseFile)); - expectedResponse = include(expectedResponse); - expectedResponse = expectedResponse.replaceAll("(\"[^\"]*\")|\\s*", "$1"); // Remove whitespace - String responseString = container.handleRequest(request).getBodyAsString(); - if (expectedResponse.contains("(ignore)")) { - // Convert expected response to a literal pattern and replace any ignored field with a pattern that matches - // until the first stop character - String stopCharacters = "[^,:\\\\[\\\\]{}]"; - String expectedResponsePattern = Pattern.quote(expectedResponse) - .replaceAll("\"?\\(ignore\\)\"?", "\\\\E" + - stopCharacters + "*\\\\Q"); - if (!Pattern.matches(expectedResponsePattern, responseString)) { - throw new ComparisonFailure(responseFile + " (with ignored fields)", expectedResponsePattern, - responseString); - } - } else { - assertEquals(responseFile, expectedResponse, responseString); - } - } - - private void assertRestart(int restartCount, Request request) throws IOException { - assertResponse(request, 200, "{\"message\":\"Scheduled restart of " + restartCount + " matching nodes\"}"); - } - - private void assertReboot(int rebootCount, Request request) throws IOException { - assertResponse(request, 200, "{\"message\":\"Scheduled reboot of " + rebootCount + " matching nodes\"}"); - } - - /** Replaces @include(localFile) with the content of the file */ - private String include(String response) throws IOException { - // Please don't look at this code - int includeIndex = response.indexOf("@include("); - if (includeIndex < 0) return response; - String prefix = response.substring(0, includeIndex); - String rest = response.substring(includeIndex + "@include(".length()); - int filenameEnd = rest.indexOf(")"); - String includeFileName = rest.substring(0, filenameEnd); - String includedContent = IOUtils.readFile(new File(responsesPath + includeFileName)); - includedContent = include(includedContent); - String postFix = rest.substring(filenameEnd + 1); - postFix = include(postFix); - return prefix + includedContent + postFix; - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json deleted file mode 100644 index 55891309856..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json +++ /dev/null @@ -1,237 +0,0 @@ -{ - "trustedNodes": [ - { - "hostname": "cfg1.yahoo.com", - "type": "config", - "ipAddress": "127.0.201.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "cfg1.yahoo.com", - "type": "config", - "ipAddress": "::201:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "cfg2.yahoo.com", - "type": "config", - "ipAddress": "127.0.202.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "cfg2.yahoo.com", - "type": "config", - "ipAddress": "::202:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "dockerhost1.yahoo.com", - "type": "host", - "ipAddress": "127.0.100.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "dockerhost1.yahoo.com", - "type": "host", - "ipAddress": "::100:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "dockerhost2.yahoo.com", - "type": "host", - "ipAddress": "127.0.101.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "dockerhost2.yahoo.com", - "type": "host", - "ipAddress": "::101:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "dockerhost3.yahoo.com", - "type": "host", - "ipAddress": "127.0.102.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "dockerhost3.yahoo.com", - "type": "host", - "ipAddress": "::102:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "dockerhost4.yahoo.com", - "type": "host", - "ipAddress": "127.0.103.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "dockerhost4.yahoo.com", - "type": "host", - "ipAddress": "::103:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "dockerhost5.yahoo.com", - "type": "host", - "ipAddress": "127.0.104.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "dockerhost5.yahoo.com", - "type": "host", - "ipAddress": "::104:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host1.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.1.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host1.yahoo.com", - "type": "tenant", - "ipAddress": "::1:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host10.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.10.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host10.yahoo.com", - "type": "tenant", - "ipAddress": "::10:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host13.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.13.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host13.yahoo.com", - "type": "tenant", - "ipAddress": "::13:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host14.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.14.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host14.yahoo.com", - "type": "tenant", - "ipAddress": "::14:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host2.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.2.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host2.yahoo.com", - "type": "tenant", - "ipAddress": "::2:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host3.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.3.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host3.yahoo.com", - "type": "tenant", - "ipAddress": "::3:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host4.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.4.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host4.yahoo.com", - "type": "tenant", - "ipAddress": "::4:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host5.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.5.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host5.yahoo.com", - "type": "tenant", - "ipAddress": "::5:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host55.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.55.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host55.yahoo.com", - "type": "tenant", - "ipAddress": "::55:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host6.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.6.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host6.yahoo.com", - "type": "tenant", - "ipAddress": "::6:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host7.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.7.1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "host7.yahoo.com", - "type": "tenant", - "ipAddress": "::7:1", - "trustedBy": "cfg1.yahoo.com" - }, - { - "hostname": "test-node-pool-102-2", - "type": "tenant", - "ipAddress": "::102:2", - "trustedBy": "cfg1.yahoo.com" - } - ], - "trustedNetworks": [], - "trustedPorts": [ - { - "port":22, - "trustedBy":"cfg1.yahoo.com" - }, - { - "port": 4443, - "trustedBy": "cfg1.yahoo.com" - } - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json deleted file mode 100644 index 16cf5c36dcc..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "trustedNodes": [ - { - "hostname": "cfg1.yahoo.com", - "type": "config", - "ipAddress": "127.0.201.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "cfg1.yahoo.com", - "type": "config", - "ipAddress": "::201:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "cfg2.yahoo.com", - "type": "config", - "ipAddress": "127.0.202.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "cfg2.yahoo.com", - "type": "config", - "ipAddress": "::202:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "foo.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.222.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host1.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.1.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host1.yahoo.com", - "type": "tenant", - "ipAddress": "::1:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host10.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.10.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host10.yahoo.com", - "type": "tenant", - "ipAddress": "::10:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host13.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.13.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host13.yahoo.com", - "type": "tenant", - "ipAddress": "::13:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host14.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.14.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host14.yahoo.com", - "type": "tenant", - "ipAddress": "::14:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host2.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.2.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host2.yahoo.com", - "type": "tenant", - "ipAddress": "::2:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host3.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.3.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host3.yahoo.com", - "type": "tenant", - "ipAddress": "::3:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host4.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.4.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host4.yahoo.com", - "type": "tenant", - "ipAddress": "::4:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host5.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.5.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host5.yahoo.com", - "type": "tenant", - "ipAddress": "::5:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host55.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.55.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host55.yahoo.com", - "type": "tenant", - "ipAddress": "::55:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host6.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.6.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host6.yahoo.com", - "type": "tenant", - "ipAddress": "::6:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host7.yahoo.com", - "type": "tenant", - "ipAddress": "127.0.7.1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "host7.yahoo.com", - "type": "tenant", - "ipAddress": "::7:1", - "trustedBy": "foo.yahoo.com" - }, - { - "hostname": "test-node-pool-102-2", - "type": "tenant", - "ipAddress": "::102:2", - "trustedBy": "foo.yahoo.com" - } - ], - "trustedNetworks": [], - "trustedPorts": [ - { - "port":22, - "trustedBy":"foo.yahoo.com" - } - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json deleted file mode 100644 index c61a755d989..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "nodes": [ - @include(node6.json), - @include(docker-container1.json), - @include(node13.json), - @include(node2.json), - @include(node14.json), - @include(node4.json), - @include(docker-node4.json), - @include(docker-node5.json), - @include(docker-node2.json), - @include(docker-node1.json), - @include(docker-node3.json) - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application1.json deleted file mode 100644 index ce2309e4d6c..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application1.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "url" : "http://localhost:8080/nodes/v2/applications/tenant1.application1.instance1", - "id" : "tenant1.application1.instance1", - "clusters" : { - "id1" : { - "min" : { - "nodes" : 2, - "groups" : 1, - "resources" : { - "vcpu" : 2.0, - "memoryGb" : 8.0, - "diskGb" : 50.0, - "bandwidthGbps" : 1.0, - "diskSpeed" : "fast", - "storageType" : "any" - } - }, - "max" : { - "nodes" : 8, - "groups" : 2, - "resources" : { - "vcpu" : 4.0, - "memoryGb" : 16.0, - "diskGb" : 1000.0, - "bandwidthGbps" : 1.0, - "diskSpeed" : "fast", - "storageType" : "any" - } - }, - "suggested" : { - "nodes" : 6, - "groups" : 2, - "resources" : { - "vcpu" : 3.0, - "memoryGb" : 20.0, - "diskGb" : 100.0, - "bandwidthGbps" : 1.0, - "diskSpeed" : "fast", - "storageType" : "any" - } - }, - "target" : { - "nodes" : 4, - "groups" : 1, - "resources" : { - "vcpu" : 3.0, - "memoryGb" : 16.0, - "diskGb" : 100.0, - "bandwidthGbps" : 1.0, - "diskSpeed" : "fast", - "storageType" : "any" - } - } - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json deleted file mode 100644 index 4581ecba73d..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "nodes": [ - @include(node6.json), - @include(node2.json) - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2.json deleted file mode 100644 index 75af5d4a328..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/applications/tenant2.application2.instance2", - "id": "tenant2.application2.instance2", - "clusters": { - "id2": { - "min": { - "nodes": 2, - "groups": 1, - "resources": { - "vcpu": 2.0, - "memoryGb": 8.0, - "diskGb": 50.0, - "bandwidthGbps": 1.0, - "diskSpeed": "fast", - "storageType": "any" - } - }, - "max": { - "nodes": 2, - "groups": 1, - "resources": { - "vcpu": 2.0, - "memoryGb": 8.0, - "diskGb": 50.0, - "bandwidthGbps": 1.0, - "diskSpeed": "fast", - "storageType": "any" - } - } - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/applications.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/applications.json deleted file mode 100644 index 6d25b3e59f5..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/applications.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "applications" : [ - { - "url" : "http://localhost:8080/nodes/v2/applications/tenant4.application4.instance4", - "id" : "tenant4.application4.instance4" - }, - { - "url" : "http://localhost:8080/nodes/v2/applications/tenant2.application2.instance2", - "id" : "tenant2.application2.instance2" - }, - { - "url" : "http://localhost:8080/nodes/v2/applications/tenant3.application3.instance3", - "id" : "tenant3.application3.instance3" - }, - { - "url" : "http://localhost:8080/nodes/v2/applications/tenant1.application1.instance1", - "id" : "tenant1.application1.instance1" - } - ] -} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-impossible.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-impossible.json deleted file mode 100644 index f85e0dcc270..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-impossible.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "hostsToRemove": [ - "dockerhost1.yahoo.com", - "dockerhost2.yahoo.com", - "dockerhost3.yahoo.com", - "dockerhost4.yahoo.com" - ], - "removalPossible": false, - "history": [ - { - "tenant": "host4.yahoo.com", - "newParent": "dockerhost5.yahoo.com", - "eligibleParents": 1 - }, - { - "tenant": "test-node-pool-102-2", - "eligibleParents": 0 - } - ] -} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-possible.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-possible.json deleted file mode 100644 index 177e1f317f7..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-possible.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "hostsToRemove": [ - "dockerhost1.yahoo.com", - "dockerhost2.yahoo.com", - "dockerhost3.yahoo.com" - ], - "removalPossible": true, - "history": [ - { - "tenant": "host4.yahoo.com", - "newParent": "dockerhost4.yahoo.com", - "eligibleParents": 1 - }, - { - "tenant": "test-node-pool-102-2", - "newParent": "dockerhost5.yahoo.com", - "eligibleParents": 1 - } - ] -} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-zone.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-zone.json deleted file mode 100644 index f6b2f96023a..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-zone.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "totalHosts": 5, - "couldLoseHosts": 4, - "failedTenantParent": "dockerhost1.yahoo.com", - "failedTenant": "host4.yahoo.com", - "failedTenantResources": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "failedTenantAllocation": "allocated to tenant3.application3.instance3 as 'content/id3/0/0'", - "hostCandidateRejectionReasons": { - "singularReasonFailures": { - "insufficientVcpu": 0, - "insufficientMemoryGb": 0, - "insufficientDiskGb": 0, - "incompatibleDiskSpeed": 0, - "insufficientAvailableIps": 0, - "violatesParentHostPolicy": 1 - }, - "totalFailures": { - "insufficientVcpu": 0, - "insufficientMemoryGb": 0, - "insufficientDiskGb": 0, - "incompatibleDiskSpeed": 0, - "insufficientAvailableIps": 0, - "violatesParentHostPolicy": 1 - } - }, - "details": { - "hostsToRemove": [ - "dockerhost3.yahoo.com", - "dockerhost1.yahoo.com", - "dockerhost4.yahoo.com", - "dockerhost5.yahoo.com" - ], - "removalPossible": false, - "history": [ - { - "tenant": "test-node-pool-102-2", - "newParent": "dockerhost2.yahoo.com", - "eligibleParents": 1 - }, - { - "tenant": "host4.yahoo.com", - "eligibleParents": 0 - } - ] - } -} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json deleted file mode 100644 index d2d39b06161..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/cfg1.yahoo.com", - "id": "cfg1.yahoo.com", - "state": "ready", - "type": "config", - "hostname": "cfg1.yahoo.com", - "openStackId": "cfg1", - "flavor": "default", - "cpuCores": 2.0, - "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.201.1", - "::201:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json deleted file mode 100644 index 969c9a1b9c2..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/cfg2.yahoo.com", - "id": "cfg2.yahoo.com", - "state": "ready", - "type": "config", - "hostname": "cfg2.yahoo.com", - "openStackId": "cfg2", - "flavor": "default", - "cpuCores": 2.0, - "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.202.1", - "::202:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json deleted file mode 100644 index dae92aae091..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "nodes": [ - @include(node4.json) - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json deleted file mode 100644 index e84f17dc40d..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "nodes": [ - @include(node6.json), - @include(docker-container1.json), - @include(node2.json), - @include(node4.json) - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json deleted file mode 100644 index 035e15ead49..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/controller1.yahoo.com", - "id": "controller1.yahoo.com", - "state": "provisioned", - "type": "controller", - "hostname": "controller1.yahoo.com", - "openStackId": "fake-controller1.yahoo.com", - "flavor": "default", - "cpuCores": 2.0, - "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.0.1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json deleted file mode 100644 index c97509b3b3d..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/test-node-pool-102-2", - "id": "test-node-pool-102-2", - "state": "active", - "type": "tenant", - "hostname": "test-node-pool-102-2", - "parentHostname": "dockerhost3.yahoo.com", - "openStackId": "fake-test-node-pool-102-2", - "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote]", - "resources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant3", - "application": "application3", - "instance": "instance3" - }, - "membership": { - "clustertype": "content", - "clusterid": "id3", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0,"diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "reserved", - "at": 123, - "agent": "system" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "::102:2" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json deleted file mode 100644 index 77622933866..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - "id": "dockerhost1.yahoo.com", - "state": "active", - "type": "host", - "hostname": "dockerhost1.yahoo.com", - "openStackId": "dockerhost1", - "flavor": "large", - "cpuCores": 4.0, - "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "owner": { - "tenant": "zoneapp", - "application": "zoneapp", - "instance": "zoneapp" - }, - "membership": { - "clustertype": "container", - "clusterid": "node-admin", - "group": "0", - "index": 0, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed": "fast", "storageType":"remote" }, - "allowedToBeDown": false, - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "currentOsVersion": "7.5.2", - "wantedOsVersion": "7.5.2", - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - }, - { - "event": "osUpgraded", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.100.1", - "::100:1" - ], - "additionalIpAddresses": [ - "::100:2", - "::100:3", - "::100:4" - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json deleted file mode 100644 index a1775dc794e..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - "id": "dockerhost1.yahoo.com", - "state": "active", - "type": "host", - "hostname": "dockerhost1.yahoo.com", - "openStackId": "dockerhost1", - "flavor": "large", - "cpuCores": 4.0, - "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast"}, - "environment": "BARE_METAL", - "owner": { - "tenant": "zoneapp", - "application": "zoneapp", - "instance": "zoneapp" - }, - "membership": { - "clustertype": "container", - "clusterid": "node-admin", - "group": "0", - "index": 0, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "currentOsVersion": "7.5.2", - "wantedOsVersion": "7.5.2", - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.100.1", - "::100:1" - ], - "additionalIpAddresses": [ - "::100:2", - "::100:3", - "::100:4" - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json deleted file mode 100644 index b6b93c8d108..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - "id": "dockerhost1.yahoo.com", - "state": "active", - "type": "host", - "hostname": "dockerhost1.yahoo.com", - "openStackId": "dockerhost1", - "flavor": "large", - "cpuCores": 4.0, - "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "owner": { - "tenant": "zoneapp", - "application": "zoneapp", - "instance": "zoneapp" - }, - "membership": { - "clustertype": "container", - "clusterid": "node-admin", - "group": "0", - "index": 0, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, - "allowedToBeDown": false, - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.100.1", - "::100:1" - ], - "additionalIpAddresses": [ - "::100:2", - "::100:3", - "::100:4" - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json deleted file mode 100644 index 88e4d132ed5..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", - "id": "dockerhost2.yahoo.com", - "state": "active", - "type": "host", - "hostname": "dockerhost2.yahoo.com", - "openStackId": "dockerhost2", - "flavor": "large", - "cpuCores": 4.0, - "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "owner": { - "tenant": "zoneapp", - "application": "zoneapp", - "instance": "zoneapp" - }, - "membership": { - "clustertype": "container", - "clusterid": "node-admin", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, - "allowedToBeDown": false, - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.101.1", - "::101:1" - ], - "additionalIpAddresses": [ - "::101:2", - "::101:3", - "::101:4" - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json deleted file mode 100644 index c890f1e72be..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com", - "id": "dockerhost3.yahoo.com", - "state": "active", - "type": "host", - "hostname": "dockerhost3.yahoo.com", - "openStackId": "dockerhost3", - "flavor": "large", - "cpuCores": 4.0, - "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "owner": { - "tenant": "zoneapp", - "application": "zoneapp", - "instance": "zoneapp" - }, - "membership": { - "clustertype": "container", - "clusterid": "node-admin", - "group": "0", - "index": 2, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, - "allowedToBeDown": false, - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.102.1", - "::102:1" - ], - "additionalIpAddresses": [ - "::102:2", - "::102:3", - "::102:4" - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json deleted file mode 100644 index 197a8c248e9..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com", - "id": "dockerhost4.yahoo.com", - "state": "active", - "type": "host", - "hostname": "dockerhost4.yahoo.com", - "openStackId": "dockerhost4", - "flavor": "large", - "cpuCores": 4.0, - "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "owner": { - "tenant": "zoneapp", - "application": "zoneapp", - "instance": "zoneapp" - }, - "membership": { - "clustertype": "container", - "clusterid": "node-admin", - "group": "0", - "index": 3, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, - "allowedToBeDown": false, - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.103.1", - "::103:1" - ], - "additionalIpAddresses": [ - "::103:2", - "::103:3", - "::103:4" - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json deleted file mode 100644 index 2638b6b74f5..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com", - "id": "dockerhost5.yahoo.com", - "state": "active", - "type": "host", - "hostname": "dockerhost5.yahoo.com", - "openStackId": "dockerhost5", - "flavor": "large", - "cpuCores": 4.0, - "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "owner": { - "tenant": "zoneapp", - "application": "zoneapp", - "instance": "zoneapp" - }, - "membership": { - "clustertype": "container", - "clusterid": "node-admin", - "group": "0", - "index": 4, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, - "allowedToBeDown": false, - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.104.1", - "::104:1" - ], - "additionalIpAddresses": [ - "::104:2", - "::104:3", - "::104:4" - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json deleted file mode 100644 index ea5c3623303..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - "id": "dockerhost1.yahoo.com", - "state": "active", - "type": "host", - "hostname": "dockerhost1.yahoo.com", - "openStackId": "dockerhost1", - "flavor": "large", - "cpuCores": 4.0, - "resources":{"vcpu":4.0,"memoryGb":32.0,"diskGb":1600.0,"bandwidthGbps":20.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "owner": { - "tenant": "zoneapp", - "application": "zoneapp", - "instance": "zoneapp" - }, - "membership": { - "clustertype": "container", - "clusterid": "node-admin", - "group": "0", - "index": 0, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast", "storageType":"remote" }, - "allowedToBeDown": false, - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "currentFirmwareCheck": 100, - "wantedFirmwareCheck": 123, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - }, - { - "event": "firmwareVerified", - "at": 100, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.100.1", - "::100:1" - ], - "additionalIpAddresses": [ - "::100:2", - "::100:3", - "::100:4" - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json deleted file mode 100644 index a606777e9fd..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "flags": [ - { - "id": "exclusive-load-balancer", - "enabled": false, - "enabledHostnames": [], - "enabledApplications": ["zoneapp:zoneapp:zoneapp"] - } - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json deleted file mode 100644 index 4baf75f2169..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "flags": [ - { - "id": "exclusive-load-balancer", - "enabled": false, - "enabledHostnames": [ - "host1" - ], - "enabledApplications": [ - "zoneapp:zoneapp:zoneapp", - "foo:bar:default" - ] - } - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers-single.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers-single.json deleted file mode 100644 index a9a728bab15..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers-single.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "loadBalancers": [ - { - "id": "tenant4:application4:instance4:id4", - "state": "active", - "changedAt": 123, - "application": "application4", - "tenant": "tenant4", - "instance": "instance4", - "cluster": "id4", - "hostname": "lb-tenant4.application4.instance4-id4", - "dnsZone": "zone-id-1", - "networks": [ - "10.2.3.0/24", - "10.4.5.0/24" - ], - "ports": [ - 4443 - ], - "reals": [ - { - "hostname": "host13.yahoo.com", - "ipAddress": "127.0.13.1", - "port": 4443 - }, - { - "hostname": "host14.yahoo.com", - "ipAddress": "127.0.14.1", - "port": 4443 - } - ] - } - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json deleted file mode 100644 index 515081bcb8e..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "loadBalancers": [ - { - "id": "tenant1:application1:instance1:id1", - "state": "reserved", - "changedAt": 123, - "application": "application1", - "tenant": "tenant1", - "instance": "instance1", - "cluster": "id1", - "hostname": "lb-tenant1.application1.instance1-id1", - "dnsZone": "zone-id-1", - "networks": [ - "10.2.3.0/24", - "10.4.5.0/24" - ], - "ports": [ - 4443 - ], - "reals": [ - { - "hostname": "host1.yahoo.com", - "ipAddress": "127.0.1.1", - "port": 4443 - }, - { - "hostname": "host10.yahoo.com", - "ipAddress": "127.0.10.1", - "port": 4443 - } - ] - }, - { - "id": "tenant4:application4:instance4:id4", - "state": "active", - "changedAt": 123, - "application": "application4", - "tenant": "tenant4", - "instance": "instance4", - "cluster": "id4", - "hostname": "lb-tenant4.application4.instance4-id4", - "dnsZone": "zone-id-1", - "networks": [ - "10.2.3.0/24", - "10.4.5.0/24" - ], - "ports": [ - 4443 - ], - "reals": [ - { - "hostname": "host13.yahoo.com", - "ipAddress": "127.0.13.1", - "port": 4443 - }, - { - "hostname": "host14.yahoo.com", - "ipAddress": "127.0.14.1", - "port": 4443 - } - ] - } - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json deleted file mode 100644 index e041a7b8b54..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "jobs": [ - { - "name": "AutoscalingMaintainer" - }, - { - "name": "CapacityReportMaintainer" - }, - { - "name": "DirtyExpirer" - }, - { - "name": "FailedExpirer" - }, - { - "name": "InactiveExpirer" - }, - { - "name": "InfrastructureProvisioner" - }, - { - "name": "LoadBalancerExpirer" - }, - { - "name": "MetricsReporter" - }, - { - "name": "NodeFailer" - }, - { - "name": "NodeMetricsDbMaintainer" - }, - { - "name": "NodeRebooter" - }, - { - "name": "OperatorChangeApplicationMaintainer" - }, - { - "name": "OsUpgradeActivator" - }, - { - "name": "PeriodicApplicationMaintainer" - }, - { - "name": "ProvisionedExpirer" - }, - { - "name": "Rebalancer" - }, - { - "name": "ReservationExpirer" - }, - { - "name": "RetiredExpirer" - }, - { - "name":"ScalingSuggestionsMaintainer" - } - ], - "inactive": [ - "NodeFailer" - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json deleted file mode 100644 index 7f6a6e17fa1..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com", - "id": "host1.yahoo.com", - "state": "reserved", - "type": "tenant", - "hostname": "host1.yahoo.com", - "openStackId": "node1", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant1", - "application": "application1", - "instance": "instance1" - }, - "membership": { - "clustertype": "container", - "clusterid": "id1", - "group": "0", - "index": 0, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.1.1", - "::1:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json deleted file mode 100644 index da0e7b28f3e..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com", - "id": "host10.yahoo.com", - "state": "reserved", - "type": "tenant", - "hostname": "host10.yahoo.com", - "parentHostname": "parent1.yahoo.com", - "openStackId": "node10", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant1", - "application": "application1", - "instance": "instance1" - }, - "membership": { - "clustertype": "container", - "clusterid": "id1", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "vespaVersion": "5.104.142", - "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:5.104.142", - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.10.1", - "::10:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json deleted file mode 100644 index 17f8210fa4d..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host11.yahoo.com", - "id": "host11.yahoo.com", - "state": "provisioned", - "type": "tenant", - "hostname": "host11.yahoo.com", - "parentHostname": "parent.host.yahoo.com", - "openStackId": "host11.yahoo.com", - "flavor": "[vcpu: 1.0, memory: 1.0 Gb, disk 100.0 Gb, bandwidth: 0.3 Gbps]", - "resources":{"vcpu":1.0,"memoryGb":1.0,"diskGb":100.0,"bandwidthGbps":0.3,"diskSpeed":"fast","storageType":"any"}, - "environment": "DOCKER_CONTAINER", - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "::11" - ], - "additionalIpAddresses": [ - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json deleted file mode 100644 index 2e96c4c9bbc..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host13.yahoo.com", - "id": "host13.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host13.yahoo.com", - "openStackId": "node13", - "flavor": "[vcpu: 10.0, memory: 48.0 Gb, disk 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":10.0,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant4", - "application": "application4", - "instance": "instance4" - }, - "membership": { - "clustertype": "container", - "clusterid": "id4", - "group": "0", - "index": 0, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.13.1", - "::13:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json deleted file mode 100644 index bfbecc09c60..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host14.yahoo.com", - "id": "host14.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host14.yahoo.com", - "openStackId": "node14", - "flavor": "[vcpu: 10.0, memory: 48.0 Gb, disk 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":10.0,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0, "diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant4", - "application": "application4", - "instance": "instance4" - }, - "membership": { - "clustertype": "container", - "clusterid": "id4", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.14.1", - "::14:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json deleted file mode 100644 index b8ce5c86c4d..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com", - "id": "host2.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host2.yahoo.com", - "openStackId": "node2", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" - }, - "membership": { - "clustertype": "content", - "clusterid": "id2", - "group": "0", - "index": 0, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.2.1", - "::2:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json deleted file mode 100644 index 70ae2a030bd..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com", - "id": "host3.yahoo.com", - "state": "ready", - "type": "tenant", - "hostname": "host3.yahoo.com", - "openStackId": "node3", - "flavor": "[vcpu: 0.5, memory: 48.0 Gb, disk 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":0.5,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.3.1", - "::3:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json deleted file mode 100644 index af3552945d9..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", - "id": "host4.yahoo.com", - "state": "failed", - "type": "tenant", - "hostname": "host4.yahoo.com", - "parentHostname": "parent.yahoo.com", - "openStackId": "patched-openstackid", - "flavor": "d-2-8-100", - "cpuCores": 2.0, - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant3", - "application": "application3", - "instance": "instance3" - }, - "membership": { - "clustertype": "content", - "clusterid": "id3", - "group": "0", - "index": 0, - "retired": false - }, - "restartGeneration": 1, - "currentRestartGeneration": 1, - "wantedDockerImage": "docker.domain.tld/my/image:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": true, - "suspendedSinceMillis": 0, - "rebootGeneration": 3, - "currentRebootGeneration": 1, - "vespaVersion": "6.43.0", - "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.45.0", - "failCount": 1, - "wantToRetire": true, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - }, - { - "event": "failed", - "at": 123, - "agent": "operator" - }, - { - "event": "rebooted", - "at": 123, - "agent": "system" - }, - { - "event": "wantToRetire", - "at": 123, - "agent": "operator" - } - ], - "ipAddresses": [ - "127.0.0.1", - "::1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json deleted file mode 100644 index c8680e1b420..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com", - "id": "host4.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host4.yahoo.com", - "parentHostname": "dockerhost1.yahoo.com", - "openStackId": "node4", - "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant3", - "application": "application3", - "instance": "instance3" - }, - "membership": { - "clustertype": "content", - "clusterid": "id3", - "group": "0", - "index": 0, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "vespaVersion": "6.41.0", - "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.41.0", - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.4.1", - "::4:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json deleted file mode 100644 index beb42ec6907..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com", - "id": "host5.yahoo.com", - "state": "failed", - "type": "tenant", - "hostname": "host5.yahoo.com", - "parentHostname": "dockerhost2.yahoo.com", - "openStackId": "node5", - "flavor": "[vcpu: 1.0, memory: 8.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, disk speed: slow, storage type: remote]", - "resources":{"vcpu":1.0,"memoryGb":8.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"slow","storageType":"remote"}, - "environment": "DOCKER_CONTAINER", - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 1, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "failed", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.5.1", - "::5:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json deleted file mode 100644 index 3fb5ffa4708..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com", - "id": "host5.yahoo.com", - "state": "failed", - "type": "tenant", - "hostname": "host5.yahoo.com", - "parentHostname": "dockerhost2.yahoo.com", - "openStackId": "node5", - "flavor": "[vcpu: 1.0, memory: 8.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, disk speed: slow, storage type: remote]", - "resources":{"vcpu":1.0,"memoryGb":8.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"slow","storageType":"remote"}, - "environment": "DOCKER_CONTAINER", - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "vespaVersion": "1.2.3", - "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:1.2.3", - "failCount": 1, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "failed", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.5.1", - "::5:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json deleted file mode 100644 index 3107311b792..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com", - "id": "host55.yahoo.com", - "state": "dirty", - "type": "tenant", - "hostname": "host55.yahoo.com", - "openStackId": "node55", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": true, - "wantToDeprovision": true, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "deallocated", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.55.1", - "::55:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json deleted file mode 100644 index 65c7e9db6cd..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":1.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" - }, - "membership": { - "clustertype": "content", - "clusterid": "id2", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.6.1", - "::6:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json deleted file mode 100644 index a3d53798d7c..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" - }, - "membership": { - "clustertype": "content", - "clusterid": "id2", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": true, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.6.1", - "::6:1" - ], - "additionalIpAddresses": [], - "reports": { - "actualCpuCores": { - "createdMillis": 3 - }, - "diskSpace": { - "createdMillis": 2, - "description": "Actual disk space (2TB) differs from spec (3TB)", - "type":"HARD_FAIL", - "details": { - "inGib": 3, - "disks": [ - "/dev/sda1", - "/dev/sdb3" - ] - } - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json deleted file mode 100644 index 7f0c3a5f706..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" - }, - "membership": { - "clustertype": "content", - "clusterid": "id2", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.6.1", - "::6:1" - ], - "additionalIpAddresses": [], - "reports": { - "actualCpuCores": { - "createdMillis": 3 - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json deleted file mode 100644 index 67b8d67c7f1..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" - }, - "membership": { - "clustertype": "content", - "clusterid": "id2", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": true, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.6.1", - "::6:1" - ], - "additionalIpAddresses": [], - "reports": { - "actualCpuCores": { - "createdMillis": 1, - "description": "Actual number of CPU cores (2) differs from spec (4)", - "type":"HARD_FAIL", - "value": 2 - }, - "diskSpace": { - "createdMillis": 2, - "description": "Actual disk space (2TB) differs from spec (3TB)", - "type":"HARD_FAIL", - "details": { - "inGib": 3, - "disks": [ - "/dev/sda1", - "/dev/sdb3" - ] - } - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json deleted file mode 100644 index 5a665a5a223..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com", - "id": "host6.yahoo.com", - "state": "active", - "type": "tenant", - "hostname": "host6.yahoo.com", - "openStackId": "node6", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "owner": { - "tenant": "tenant2", - "application": "application2", - "instance": "instance2" - }, - "membership": { - "clustertype": "content", - "clusterid": "id2", - "group": "0", - "index": 1, - "retired": false - }, - "restartGeneration": 0, - "currentRestartGeneration": 0, - "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" }, - "allowedToBeDown": false, - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - }, - { - "event": "reserved", - "at": 123, - "agent": "application" - }, - { - "event": "activated", - "at": 123, - "agent": "application" - } - ], - "ipAddresses": [ - "127.0.6.1", - "::6:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json deleted file mode 100644 index 38757b81156..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com", - "id": "host7.yahoo.com", - "state": "provisioned", - "type": "tenant", - "hostname": "host7.yahoo.com", - "openStackId": "node7", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local]", - "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local"}, - "environment": "DOCKER_CONTAINER", - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.7.1", - "::7:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json deleted file mode 100644 index aa2b2acdb9f..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host8.yahoo.com", - "id": "host8.yahoo.com", - "state": "provisioned", - "type": "tenant", - "hostname": "host8.yahoo.com", - "openStackId": "host8.yahoo.com", - "flavor": "default", - "cpuCores": 2.0, - "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":10.0,"diskSpeed":"fast","storageType": "remote"}, - "environment": "BARE_METAL", - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.8.1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json deleted file mode 100644 index 5f388c13f28..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/host9.yahoo.com", - "id": "host9.yahoo.com", - "state": "provisioned", - "type": "tenant", - "hostname": "host9.yahoo.com", - "openStackId": "host9.yahoo.com", - "flavor": "large-variant", - "cpuCores": 64.0, - "resources":{"vcpu":64.0,"memoryGb":128.0,"diskGb":2000.0,"bandwidthGbps":15.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.9.1", - "::9:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json deleted file mode 100644 index cf3e3c1b457..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "nodes": [ - @include(node7.json), - @include(cfg1.json), - @include(node3.json), - @include(cfg2.json), - @include(node10.json), - @include(node1.json), - @include(docker-node4.json), - @include(node6.json), - @include(docker-container1.json), - @include(docker-node5.json), - @include(docker-node2.json), - @include(node13.json), - @include(node2.json), - @include(docker-node1.json), - @include(docker-node3.json), - @include(node14.json), - @include(node4.json), - @include(node55.json), - @include(node5.json) - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json deleted file mode 100644 index cdbd2cac9b9..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "nodes": [ - { - "url": "http://localhost:8080/nodes/v2/node/host7.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/cfg1.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/host3.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/cfg2.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/host10.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/host1.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/dockerhost4.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/host6.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/test-node-pool-102-2" - }, - { - "url": "http://localhost:8080/nodes/v2/node/dockerhost5.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/host13.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/host14.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/host4.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/host55.yahoo.com" - }, - { - "url": "http://localhost:8080/nodes/v2/node/host5.yahoo.com" - } - ] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json deleted file mode 100644 index 40f4a3b5160..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/parent1.yahoo.com", - "id": "parent1.yahoo.com", - "state": "ready", - "type": "host", - "hostname": "parent1.yahoo.com", - "openStackId": "parent1", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb]", - "resources":{"vcpu":2.0,"memoryGb":16.0,"diskGb":400.0,"bandwidthGbps":1.0,"diskSpeed":"fast"}, - "environment": "BARE_METAL", - "rebootGeneration": 1, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - }, - { - "event": "readied", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.0.1", - "::1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json deleted file mode 100644 index ecc497172c7..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "url": "http://localhost:8080/nodes/v2/node/parent2.yahoo.com", - "id": "parent2.yahoo.com", - "state": "provisioned", - "type": "host", - "hostname": "parent2.yahoo.com", - "openStackId": "parent2.yahoo.com", - "flavor": "large-variant", - "reservedTo": "myTenant", - "cpuCores": 64.0, - "resources": {"vcpu":64.0,"memoryGb":128.0,"diskGb":2000.0,"bandwidthGbps":15.0,"diskSpeed":"fast","storageType":"remote"}, - "environment": "BARE_METAL", - "rebootGeneration": 0, - "currentRebootGeneration": 0, - "failCount": 0, - "wantToRetire": false, - "wantToDeprovision": false, - "history": [ - { - "event": "provisioned", - "at": 123, - "agent": "system" - } - ], - "ipAddresses": [ - "127.0.127.1", - "::127:1" - ], - "additionalIpAddresses": [] -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json deleted file mode 100644 index cc91314df84..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "resources": [ - { - "url": "http://localhost:8080/nodes/v2/state/" - }, - { - "url": "http://localhost:8080/nodes/v2/node/" - }, - { - "url": "http://localhost:8080/nodes/v2/command/" - }, - { - "url": "http://localhost:8080/nodes/v2/maintenance/" - }, - { - "url":"http://localhost:8080/nodes/v2/upgrade/" - }, - { - "url":"http://localhost:8080/nodes/v2/application/" - } - ] -} \ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json deleted file mode 100644 index f8343756559..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "states": { - "provisioned": { - "url": "http://localhost:8080/nodes/v2/state/provisioned", - "nodes": [ - @include(node7.json) - ] - }, - "ready": { - "url": "http://localhost:8080/nodes/v2/state/ready", - "nodes": [ - @include(node3.json), - @include(cfg1.json), - @include(cfg2.json) - ] - }, - "reserved": { - "url": "http://localhost:8080/nodes/v2/state/reserved", - "nodes": [ - @include(node10.json), - @include(node1.json) - ] - }, - "active": { - "url": "http://localhost:8080/nodes/v2/state/active", - "nodes": [ - @include(node6.json), - @include(docker-container1.json), - @include(node13.json), - @include(node2.json), - @include(node14.json), - @include(node4.json), - @include(docker-node4.json), - @include(docker-node5.json), - @include(docker-node2.json), - @include(docker-node1.json), - @include(docker-node3.json) - ] - }, - "inactive": { - "url": "http://localhost:8080/nodes/v2/state/inactive", - "nodes": [ - ] - }, - "dirty": { - "url": "http://localhost:8080/nodes/v2/state/dirty", - "nodes": [ - @include(node55.json) - ] - }, - "failed": { - "url": "http://localhost:8080/nodes/v2/state/failed", - "nodes": [ - @include(node5.json) - ] - }, - "parked": { - "url": "http://localhost:8080/nodes/v2/state/parked", - "nodes": [ - ] - }, - "deprovisioned": { - "url": "http://localhost:8080/nodes/v2/state/deprovisioned", - "nodes": [ - ] - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json deleted file mode 100644 index 69579148df3..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "states": { - "provisioned": { - "url": "http://localhost:8080/nodes/v2/state/provisioned" - }, - "ready": { - "url": "http://localhost:8080/nodes/v2/state/ready" - }, - "reserved": { - "url": "http://localhost:8080/nodes/v2/state/reserved" - }, - "active": { - "url": "http://localhost:8080/nodes/v2/state/active" - }, - "inactive": { - "url": "http://localhost:8080/nodes/v2/state/inactive" - }, - "dirty": { - "url": "http://localhost:8080/nodes/v2/state/dirty" - }, - "failed": { - "url": "http://localhost:8080/nodes/v2/state/failed" - }, - "parked": { - "url": "http://localhost:8080/nodes/v2/state/parked" - }, - "deprovisioned": { - "url": "http://localhost:8080/nodes/v2/state/deprovisioned" - } - } -} \ No newline at end of file -- cgit v1.2.3 From 0818b4a010cda88004cb5ad77bb7108de0fc96d8 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Sun, 26 Apr 2020 19:07:16 +0200 Subject: Handle unspecified resources --- .../provisioning/NodeRepositoryProvisioner.java | 26 ++++++++++------------ 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 434860ccb30..4000354243f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -153,13 +153,7 @@ public class NodeRepositoryProvisioner implements Provisioner { if (nodes.isEmpty()) return requested.minResources(); // New deployment: Start at min long groups = nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); - // Settings which are not autoscaled should always be taken from the currently requested capacity - // and for those min and max are always the same - NodeResources nodeResources = nodes.get(0).allocation().get().requestedResources() - .withBandwidthGbps(requested.minResources().nodeResources().bandwidthGbps()) - .with(requested.minResources().nodeResources().diskSpeed()) - .with(requested.minResources().nodeResources().storageType()); - var currentResources = new ClusterResources(nodes.size(), (int)groups, nodeResources); + var currentResources = new ClusterResources(nodes.size(), (int)groups, nodes.get(0).flavor().resources()); return ensureWithin(requested.minResources(), requested.maxResources(), currentResources); } @@ -178,13 +172,17 @@ public class NodeRepositoryProvisioner implements Provisioner { groups = max.groups(); } } - double vcpu = between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), current.nodeResources().vcpu()); - double memoryGb = between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), current.nodeResources().memoryGb()); - double diskGb = between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), current.nodeResources().diskGb()); - NodeResources nodeResources = current.nodeResources().withVcpu(vcpu) - .withMemoryGb(memoryGb) - .withDiskGb(diskGb); - return new ClusterResources(nodes, groups, nodeResources); + if (min.nodeResources() != NodeResources.unspecified && max.nodeResources() != NodeResources.unspecified) { + double vcpu = between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), current.nodeResources().vcpu()); + double memoryGb = between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), current.nodeResources().memoryGb()); + double diskGb = between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), current.nodeResources().diskGb()); + // Combine computed scaled resources with requested non-scaled resources (for which min=max) + NodeResources nodeResources = min.nodeResources().withVcpu(vcpu).withMemoryGb(memoryGb).withDiskGb(diskGb); + return new ClusterResources(nodes, groups, nodeResources); + } + else { + return new ClusterResources(nodes, groups, current.nodeResources()); + } } private int between(int min, int max, int n) { -- cgit v1.2.3 From 18d63e6c0793474e1d99bc25a607cfe27d6bcc1f Mon Sep 17 00:00:00 2001 From: gjoranv Date: Sun, 26 Apr 2020 19:39:33 +0200 Subject: Reapply "Use java.util.logging.Level instead of vespalog's LogLevel." This reverts commit 6ef94983a16650f60db15485067e0a2bbf7bc420. --- config/pom.xml | 2 +- .../subscription/CfgConfigPayloadBuilder.java | 8 +++-- .../com/yahoo/config/subscription/ConfigDebug.java | 5 ++-- .../yahoo/config/subscription/ConfigSourceSet.java | 5 ++-- .../config/subscription/ConfigSubscriber.java | 11 ++++--- .../subscription/impl/FileConfigSubscription.java | 17 ++++++----- .../impl/GenericJRTConfigSubscription.java | 5 ++-- .../subscription/impl/JRTConfigRequester.java | 35 ++++++++++++---------- .../subscription/impl/JRTConfigSubscription.java | 21 ++++++------- .../yahoo/vespa/config/ConfigPayloadApplier.java | 15 ++++++---- .../com/yahoo/vespa/config/JRTConnectionPool.java | 5 ++-- .../java/com/yahoo/vespa/config/UrlDownloader.java | 10 ++++--- .../vespa/config/protocol/RequestValidation.java | 17 ++++++----- 13 files changed, 88 insertions(+), 68 deletions(-) diff --git a/config/pom.xml b/config/pom.xml index ad036c442d1..6e4e26ed0f1 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -53,7 +53,7 @@ com.yahoo.vespa - vespalog + defaults ${project.version} diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java index 474c9f7a4db..74d4d378e5d 100644 --- a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java +++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java @@ -4,13 +4,15 @@ package com.yahoo.config.subscription; import com.yahoo.collections.Pair; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.StringNode; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; import java.util.ArrayList; import java.util.List; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.FINEST; + /** * Deserializes config payload (cfg format) to a ConfigPayload. * @@ -37,7 +39,7 @@ public class CfgConfigPayloadBuilder { parseLine(line, lineNum, payloadBuilder); lineNum++; } - log.log(LogLevel.SPAM, () -> "payload=" + payloadBuilder.toString()); + log.log(FINEST, () -> "payload=" + payloadBuilder.toString()); return payloadBuilder; } @@ -48,7 +50,7 @@ public class CfgConfigPayloadBuilder { String field = fieldAndValue.getFirst(); String value = fieldAndValue.getSecond(); if (field==null || value==null) { - log.log(LogLevel.DEBUG, () -> "Got field without value in line " + lineNum + ": " + line + ", skipping"); + log.log(FINE, () -> "Got field without value in line " + lineNum + ": " + line + ", skipping"); return; } field=field.trim(); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java index 3020f49b32e..d279fc42251 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java @@ -2,16 +2,17 @@ package com.yahoo.config.subscription; import com.yahoo.config.ConfigInstance; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import java.util.logging.Logger; +import static java.util.logging.Level.INFO; + // Debug class that provides useful helper routines public class ConfigDebug { public static void logDebug(Logger logger, long timestamp, ConfigKey key, String logmessage) { if (key.getConfigId().matches(".*container.?\\d+.*") || key.getConfigId().matches(".*doc.api.*")) { - logger.log(LogLevel.INFO, timestamp + " " + key + " " + logmessage); + logger.log(INFO, timestamp + " " + key + " " + logmessage); } } diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java index 7472439d6a4..e0605dce30d 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import com.yahoo.log.LogLevel; - import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; @@ -10,6 +8,7 @@ import java.util.List; import java.util.Set; import java.util.logging.Logger; +import static java.util.logging.Level.INFO; /** * An immutable set of connection endpoints, where each endpoint points to either a @@ -98,7 +97,7 @@ public class ConfigSourceSet implements ConfigSource { public static ConfigSourceSet createDefault() { String configSources = System.getenv("VESPA_CONFIG_SOURCES"); if (configSources != null) { - log.log(LogLevel.INFO, "Using config sources from VESPA_CONFIG_SOURCES: " + configSources); + log.log(INFO, "Using config sources from VESPA_CONFIG_SOURCES: " + configSources); return new ConfigSourceSet(checkSourcesSyntax(configSources)); } else { String[] def = {"tcp/localhost:" + System.getProperty("vespa.config.port", "19090")}; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index 3891d710fa3..814222989a1 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -5,7 +5,6 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.impl.ConfigSubscription; import com.yahoo.config.subscription.impl.JRTConfigRequester; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.TimingValues; import com.yahoo.yolean.Exceptions; @@ -14,8 +13,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; + import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.SEVERE; +import static java.util.logging.Level.WARNING; import static java.util.stream.Collectors.toList; /** @@ -326,7 +329,7 @@ public class ConfigSubscriber implements AutoCloseable { h.subscription().close(); } closeRequesters(); - log.log(LogLevel.DEBUG, () -> "Config subscriber has been closed."); + log.log(FINE, () -> "Config subscriber has been closed."); } /** @@ -434,7 +437,7 @@ public class ConfigSubscriber implements AutoCloseable { if (handle.isChanged()) singleSubscriber.configure(handle.getConfig()); } } catch (Exception e) { - log.log(LogLevel.ERROR, "Exception from config system, continuing config thread: " + Exceptions.toMessageString(e)); + log.log(SEVERE, "Exception from config system, continuing config thread: " + Exceptions.toMessageString(e)); } } } @@ -477,7 +480,7 @@ public class ConfigSubscriber implements AutoCloseable { protected void finalize() throws Throwable { try { if (!isClosed()) { - log.log(LogLevel.WARNING, stackTraceAtConstruction, + log.log(WARNING, stackTraceAtConstruction, () -> String.format("%s: Closing subscription from finalizer() - close() has not been called (keys=%s)", super.toString(), subscriptionHandles.stream().map(handle -> handle.subscription().getKey().toString()).collect(toList()))); diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java index 60c920d1dfa..365f6a4d4c5 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java @@ -1,10 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; -import java.io.File; -import java.io.IOException; -import java.util.Arrays; - import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.CfgConfigPayloadBuilder; @@ -13,7 +9,12 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.io.IOUtils; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.log.LogLevel; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import static java.util.logging.Level.FINE; /** * Subscription used when config id is file:... @@ -37,12 +38,12 @@ public class FileConfigSubscription extends ConfigSubs public boolean nextConfig(long timeout) { if (!file.exists() && !file.isFile()) throw new IllegalArgumentException("Not a file: "+file); if (checkReloaded()) { - log.log(LogLevel.DEBUG, "User forced config reload at " + System.currentTimeMillis()); + log.log(FINE, "User forced config reload at " + System.currentTimeMillis()); // User forced reload setConfigIfChanged(updateConfig()); ConfigState configState = getConfigState(); - log.log(LogLevel.DEBUG, "Config updated at " + System.currentTimeMillis() + ", changed: " + configState.isConfigChanged()); - log.log(LogLevel.DEBUG, "Config: " + configState.getConfig().toString()); + log.log(FINE, "Config updated at " + System.currentTimeMillis() + ", changed: " + configState.isConfigChanged()); + log.log(FINE, "Config: " + configState.getConfig().toString()); return true; } if (file.lastModified()!=ts) { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java index 8ce3449fba5..eec18b93e71 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -5,13 +5,14 @@ import java.util.List; import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; +import static java.util.logging.Level.FINE; + /** * A JRT subscription which does not use the config class, but {@link com.yahoo.vespa.config.RawConfig} instead. * Used by config proxy. @@ -33,7 +34,7 @@ public class GenericJRTConfigSubscription extends JRTConfigSubscription "in setNewConfig, config=" + this.getConfigState().getConfig()); + log.log(FINE, () -> "in setNewConfig, config=" + this.getConfigState().getConfig()); } // This method is overridden because config needs to have its generation diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 49c5dcd343c..51266f21846 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -6,7 +6,6 @@ import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Request; import com.yahoo.jrt.RequestWaiter; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.ErrorCode; @@ -24,6 +23,12 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.FINEST; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.SEVERE; +import static java.util.logging.Level.WARNING; + /** * This class fetches config payload using JRT, and acts as the callback target. * It uses the {@link JRTConfigSubscription} and {@link JRTClientConfigRequest} @@ -90,9 +95,9 @@ public class JRTConfigRequester implements RequestWaiter { if ( ! req.validateParameters()) throw new ConfigurationRuntimeException("Error in parameters for config request: " + req); double jrtClientTimeout = getClientTimeout(req); - log.log(LogLevel.DEBUG, () -> "Requesting config for " + sub + " on connection " + connection + log.log(FINE, () -> "Requesting config for " + sub + " on connection " + connection + " with client timeout " + jrtClientTimeout + - (log.isLoggable(LogLevel.SPAM) ? (",defcontent=" + req.getDefContent().asString()) : "")); + (log.isLoggable(FINEST) ? (",defcontent=" + req.getDefContent().asString()) : "")); connection.invokeAsync(req.getRequest(), jrtClientTimeout, this); } @@ -110,7 +115,7 @@ public class JRTConfigRequester implements RequestWaiter { sub.setException(e); } else { // Very unlikely - log.log(Level.SEVERE, "Failed to get subscription object from JRT config callback: " + + log.log(SEVERE, "Failed to get subscription object from JRT config callback: " + Exceptions.toMessageString(e)); } } @@ -118,11 +123,11 @@ public class JRTConfigRequester implements RequestWaiter { private void doHandle(JRTConfigSubscription sub, JRTClientConfigRequest jrtReq, Connection connection) { boolean validResponse = jrtReq.validateResponse(); - log.log(LogLevel.DEBUG, () -> "Request callback " + (validResponse ? "valid" : "invalid") + ". Req: " + jrtReq + "\nSpec: " + connection); + log.log(FINE, () -> "Request callback " + (validResponse ? "valid" : "invalid") + ". Req: " + jrtReq + "\nSpec: " + connection); if (sub.getState() == ConfigSubscription.State.CLOSED) return; // Avoid error messages etc. after closing Trace trace = jrtReq.getResponseTrace(); trace.trace(TRACELEVEL, "JRTConfigRequester.doHandle()"); - log.log(LogLevel.SPAM, () -> trace.toString()); + log.log(FINEST, () -> trace.toString()); if (validResponse) { handleOKRequest(jrtReq, sub, connection); } else { @@ -134,20 +139,20 @@ public class JRTConfigRequester implements RequestWaiter { private void logWhenErrorResponse(JRTClientConfigRequest jrtReq, Connection connection) { switch (jrtReq.errorCode()) { case com.yahoo.jrt.ErrorCode.CONNECTION: - log.log(LogLevel.DEBUG, () -> "Request callback failed: " + jrtReq.errorMessage() + + log.log(FINE, () -> "Request callback failed: " + jrtReq.errorMessage() + "\nConnection spec: " + connection); break; case ErrorCode.APPLICATION_NOT_LOADED: case ErrorCode.UNKNOWN_VESPA_VERSION: if (noApplicationWarningLogged.isBefore(Instant.now().minus(delayBetweenWarnings))) { - log.log(LogLevel.WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + + log.log(WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + ". Connection spec: " + connection.getAddress() + ", error message: " + jrtReq.errorMessage()); noApplicationWarningLogged = Instant.now(); } break; default: - log.log(LogLevel.WARNING, "Request callback failed. Req: " + jrtReq + "\nSpec: " + connection.getAddress() + + log.log(WARNING, "Request callback failed. Req: " + jrtReq + "\nSpec: " + connection.getAddress() + " . Req error message: " + jrtReq.errorMessage()); break; } @@ -157,7 +162,7 @@ public class JRTConfigRequester implements RequestWaiter { final boolean configured = (sub.getConfigState().getConfig() != null); if (configured) { // The subscription object has an "old" config, which is all we have to offer back now - log.log(LogLevel.INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); + log.log(INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); } ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode()); connectionPool.setError(connection, jrtReq.errorCode()); @@ -192,7 +197,7 @@ public class JRTConfigRequester implements RequestWaiter { Connection connection) { transientFailures++; if (suspendWarningLogged.isBefore(Instant.now().minus(delayBetweenWarnings))) { - log.log(LogLevel.INFO, "Connection to " + connection.getAddress() + + log.log(INFO, "Connection to " + connection.getAddress() + " failed or timed out, clients will keep existing config, will keep trying."); suspendWarningLogged = Instant.now(); } @@ -218,7 +223,7 @@ public class JRTConfigRequester implements RequestWaiter { if (sub.getState() != ConfigSubscription.State.OPEN) return; fatalFailures++; // The logging depends on whether we are configured or not. - Level logLevel = sub.getConfigState().getConfig() == null ? LogLevel.DEBUG : LogLevel.INFO; + Level logLevel = sub.getConfigState().getConfig() == null ? Level.FINE : Level.INFO; String logMessage = "Request for config " + jrtReq.getShortDescription() + "' failed with error code " + jrtReq.errorCode() + " (" + jrtReq.errorMessage() + "), scheduling new connect " + " in " + delay + " ms"; @@ -236,7 +241,7 @@ public class JRTConfigRequester implements RequestWaiter { noApplicationWarningLogged = Instant.MIN; connection.setSuccess(); sub.setLastCallBackOKTS(Instant.now()); - log.log(LogLevel.DEBUG, () -> "OK response received in handleOkRequest: " + jrtReq); + log.log(FINE, () -> "OK response received in handleOkRequest: " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { // We only want this latest generation to be in the queue, we do not preserve history in this system sub.getReqQueue().clear(); @@ -260,8 +265,8 @@ public class JRTConfigRequester implements RequestWaiter { private void scheduleNextRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription sub, long delay, long timeout) { long delayBeforeSendingRequest = (delay < 0) ? 0 : delay; JRTClientConfigRequest jrtReqNew = jrtReq.nextRequest(timeout); - log.log(LogLevel.SPAM, () -> timingValues.toString()); - log.log(LogLevel.DEBUG, () -> "Scheduling new request " + delayBeforeSendingRequest + " millis from now for " + jrtReqNew.getConfigKey()); + log.log(FINEST, () -> timingValues.toString()); + log.log(FINE, () -> "Scheduling new request " + delayBeforeSendingRequest + " millis from now for " + jrtReqNew.getConfigKey()); scheduler.schedule(new GetConfigTask(jrtReqNew, sub), delayBeforeSendingRequest, TimeUnit.MILLISECONDS); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index a94a135f9d8..44f6d65ee65 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -1,25 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.ConfigInterruptedException; import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.JRTConnectionPool; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.CompressionType; import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; import com.yahoo.vespa.config.protocol.Payload; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.INFO; + /** * A JRT config subscription uses one {@link JRTConfigRequester} to fetch config using Vespa RPC from a config source, typically proxy or server * @@ -90,7 +91,7 @@ public class JRTConfigSubscription extends ConfigSubsc // timed out, we know nothing new. return false; } - log.log(LogLevel.DEBUG, () -> "Polled queue and found config " + jrtReq); + log.log(FINE, () -> "Polled queue and found config " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { setInternalRedeploy(jrtReq.responseIsInternalRedeploy()); if (jrtReq.hasUpdatedConfig()) { @@ -192,7 +193,7 @@ public class JRTConfigSubscription extends ConfigSubsc @Override public void reload(long generation) { - log.log(LogLevel.DEBUG, "reload() is without effect on a JRTConfigSubscription."); + log.log(FINE, "reload() is without effect on a JRTConfigSubscription."); } void setLastCallBackOKTS(Instant lastCallBackOKTS) { @@ -204,7 +205,7 @@ public class JRTConfigSubscription extends ConfigSubsc static void printStatus(JRTClientConfigRequest request, String message) { final String name = request.getConfigKey().getName(); if (name.equals("components") || name.equals("chains")) { - log.log(LogLevel.INFO, message + ":" + name + ":" + ", request=" + request); + log.log(INFO, message + ":" + name + ":" + ", request=" + request); } } } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 134352736b6..b4df42c802e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -5,13 +5,12 @@ import com.yahoo.config.ConfigBuilder; import com.yahoo.config.ConfigInstance; import com.yahoo.config.FileReference; import com.yahoo.config.UrlReference; -import com.yahoo.log.LogLevel; -import com.yahoo.yolean.Exceptions; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Type; import com.yahoo.text.Utf8; +import com.yahoo.yolean.Exceptions; import java.io.File; import java.lang.reflect.Constructor; @@ -27,6 +26,10 @@ import java.util.Set; import java.util.Stack; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.FINEST; +import static java.util.logging.Level.INFO; + /** * A utility class that can be used to apply a payload to a config builder. * @@ -224,7 +227,7 @@ public class ConfigPayloadApplier { } catch (InvocationTargetException | IllegalAccessException e) { throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); } catch (NoSuchMethodException e) { - log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + rootBuilder); + log.log(INFO, "Skipping unknown field " + methodName + " in " + rootBuilder); } } @@ -279,7 +282,7 @@ public class ConfigPayloadApplier { } catch (InvocationTargetException | IllegalAccessException e) { throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); } catch (NoSuchMethodException e) { - log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + builder.getClass()); + log.log(INFO, "Skipping unknown field " + methodName + " in " + builder.getClass()); } } @@ -479,11 +482,11 @@ public class ConfigPayloadApplier { } private void debug(String message) { - log.log(LogLevel.DEBUG, () -> message); + log.log(FINE, () -> message); } private void trace(String message) { - log.log(LogLevel.SPAM, () -> message); + log.log(FINEST, () -> message); } private void printStack() { diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java index 326c1287468..2c1d00f295e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.config; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -13,6 +12,8 @@ import java.util.Map; import java.util.concurrent.ThreadLocalRandom; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; + /** * A pool of JRT connections to a config source (either a config server or a config proxy). * The current connection is chosen randomly when calling {#link #setNewCurrentConnection} @@ -74,7 +75,7 @@ public class JRTConnectionPool implements ConnectionPool { public synchronized JRTConnection setNewCurrentConnection() { List sources = getSources(); currentConnection = sources.get(ThreadLocalRandom.current().nextInt(0, sources.size())); - log.log(LogLevel.DEBUG, () -> "Choosing new connection: " + currentConnection); + log.log(FINE, () -> "Choosing new connection: " + currentConnection); return currentConnection; } diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java index 863e95c5625..2356b400343 100644 --- a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java +++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java @@ -8,12 +8,14 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.defaults.Defaults; import java.io.File; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.INFO; + /** * @author lesters */ @@ -47,7 +49,7 @@ public class UrlDownloader { Request request = new Request("frt.rpc.ping"); target.invokeSync(request, 5.0); if (! request.isError()) { - log.log(LogLevel.DEBUG, () -> "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + log.log(FINE, () -> "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); return; } else { target.close(); @@ -78,7 +80,7 @@ public class UrlDownloader { request.parameters().add(new StringValue(urlReference.value())); double rpcTimeout = Math.min(timeLeft, 60 * 60.0); - log.log(LogLevel.DEBUG, () -> "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); + log.log(FINE, () -> "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); target.invokeSync(request, rpcTimeout); if (request.checkReturnTypes("s")) { @@ -86,7 +88,7 @@ public class UrlDownloader { } else if (!request.isError()) { throw new RuntimeException("Invalid response: " + request.returnValues()); } else if (temporaryError(request)) { - log.log(LogLevel.INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); + log.log(INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); try { Thread.sleep(1000); } catch (InterruptedException e) { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java index 8c7c7f2703e..478dc6c45d6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ErrorCode; @@ -10,6 +9,8 @@ import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static java.util.logging.Level.INFO; + /** * Static utility methods for verifying common request properties. * @@ -23,31 +24,31 @@ public class RequestValidation { public static int validateRequest(JRTConfigRequest request) { ConfigKey key = request.getConfigKey(); if (!RequestValidation.verifyName(key.getName())) { - log.log(LogLevel.INFO, "Illegal name '" + key.getName() + "'"); + log.log(INFO, "Illegal name '" + key.getName() + "'"); return ErrorCode.ILLEGAL_NAME; } if (!RequestValidation.verifyNamespace(key.getNamespace())) { - log.log(LogLevel.INFO, "Illegal name space '" + key.getNamespace() + "'"); + log.log(INFO, "Illegal name space '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_NAME_SPACE; } if (!RequestValidation.verifyMd5(key.getMd5())) { - log.log(LogLevel.INFO, "Illegal md5 sum '" + key.getNamespace() + "'"); + log.log(INFO, "Illegal md5 sum '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_DEF_MD5; } if (!RequestValidation.verifyMd5(request.getRequestConfigMd5())) { - log.log(LogLevel.INFO, "Illegal config md5 '" + request.getRequestConfigMd5() + "'"); + log.log(INFO, "Illegal config md5 '" + request.getRequestConfigMd5() + "'"); return ErrorCode.ILLEGAL_CONFIG_MD5; } if (!RequestValidation.verifyGeneration(request.getRequestGeneration())) { - log.log(LogLevel.INFO, "Illegal generation '" + request.getRequestGeneration() + "'"); + log.log(INFO, "Illegal generation '" + request.getRequestGeneration() + "'"); return ErrorCode.ILLEGAL_GENERATION; } if (!RequestValidation.verifyTimeout(request.getTimeout())) { - log.log(LogLevel.INFO, "Illegal timeout '" + request.getTimeout() + "'"); + log.log(INFO, "Illegal timeout '" + request.getTimeout() + "'"); return ErrorCode.ILLEGAL_TIMEOUT; } if (!RequestValidation.verifyHostname(request.getClientHostName())) { - log.log(LogLevel.INFO, "Illegal client host name '" + request.getClientHostName() + "'"); + log.log(INFO, "Illegal client host name '" + request.getClientHostName() + "'"); return ErrorCode.ILLEGAL_CLIENT_HOSTNAME; } return 0; -- cgit v1.2.3 From 82b5946da9cc64e536793fd48790b14286c31bb2 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Sun, 26 Apr 2020 20:32:16 +0200 Subject: Remove unused imports of vespalog package --- .../java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java | 1 - .../yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java | 1 - 2 files changed, 2 deletions(-) diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 559f690a32d..9f900ffed36 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -3,7 +3,6 @@ package com.yahoo.searchlib.rankingexpression; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.yahoo.log.event.Collection; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java index f4b1b0ceee2..cf761aac2d3 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.transform; -import com.yahoo.log.event.Collection; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.Context; import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; -- cgit v1.2.3 From 26efd1a58ee880929e78819b854224e6da580019 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Sun, 26 Apr 2020 20:30:36 +0200 Subject: Add dependency to vespalog (directly used by these modules) --- clustercontroller-core/pom.xml | 8 +++++++- container-accesslogging/pom.xml | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml index a952016e3bf..ee503d91540 100644 --- a/clustercontroller-core/pom.xml +++ b/clustercontroller-core/pom.xml @@ -18,6 +18,12 @@ guava provided + + com.yahoo.vespa + vespalog + ${project.version} + test + junit junit @@ -60,7 +66,7 @@ ${project.version} provided - + com.yahoo.vespa zkfacade ${project.version} diff --git a/container-accesslogging/pom.xml b/container-accesslogging/pom.xml index 59dabba9efe..ba544e04fc8 100644 --- a/container-accesslogging/pom.xml +++ b/container-accesslogging/pom.xml @@ -50,6 +50,12 @@ ${project.version} provided + + com.yahoo.vespa + vespalog + ${project.version} + provided + org.hamcrest hamcrest-all -- cgit v1.2.3 From fb95ee2f58ec08d7aac7351841f7c421c3653620 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Sun, 26 Apr 2020 18:18:10 +0200 Subject: Use gcc-toolset-9 on CentOS 8 and RHEL 8. --- build_settings.cmake | 13 +++++++++++-- dist/vespa.spec | 5 +++-- vespamalloc/src/tests/allocfree/CMakeLists.txt | 2 +- vespamalloc/src/tests/test1/CMakeLists.txt | 2 +- vespamalloc/src/vespamalloc/CMakeLists.txt | 8 ++++---- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/build_settings.cmake b/build_settings.cmake index ac4e86c2e81..c96a626ad0b 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -54,7 +54,12 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" ST endif() else() set(CXX_SPECIFIC_WARN_OPTS "-Wsuggest-override -Wnon-virtual-dtor -Wformat-security") - set(VESPA_ATOMIC_LIB "atomic") + if(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8" OR + VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.1") + set(VESPA_ATOMIC_LIB "") + else() + set(VESPA_ATOMIC_LIB "atomic") + endif() set(VESPA_GCC_LIB "gcc") set(VESPA_STDCXX_FS_LIB "stdc++fs") endif() @@ -79,7 +84,11 @@ if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -latomic -ldl") endif() else() - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--build-id -latomic -ldl -Wl,-E") + if(VESPA_ATOMIC_LIB STREQUAL "") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--build-id -ldl -Wl,-E") + else() + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--build-id -latomic -ldl -Wl,-E") + endif() endif() SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic" ) diff --git a/dist/vespa.spec b/dist/vespa.spec index 26310c19750..aa0232fa421 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -38,9 +38,10 @@ BuildRequires: rh-maven35 %define _rhmaven35_enable /opt/rh/rh-maven35/enable %endif %if 0%{?el8} -BuildRequires: gcc-c++ -BuildRequires: libatomic +BuildRequires: gcc-toolset-9-gcc-c++ +BuildRequires: gcc-toolset-9-binutils BuildRequires: maven +%define _devtoolset_enable /opt/rh/gcc-toolset-9/enable %endif %if 0%{?fedora} BuildRequires: gcc-c++ diff --git a/vespamalloc/src/tests/allocfree/CMakeLists.txt b/vespamalloc/src/tests/allocfree/CMakeLists.txt index 52c0c03d079..34d402d0c60 100644 --- a/vespamalloc/src/tests/allocfree/CMakeLists.txt +++ b/vespamalloc/src/tests/allocfree/CMakeLists.txt @@ -21,7 +21,7 @@ vespa_add_executable(vespamalloc_linklist_test_app $ DEPENDS dl - atomic + ${VESPA_ATOMIC_LIB} ) vespa_add_test(NAME vespamalloc_allocfree_shared_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/allocfree_test.sh BENCHMARK DEPENDS vespamalloc_realloc_test_app vespamalloc_allocfree_shared_test_app vespamalloc_linklist_test_app diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index cbfb9543cda..cade2e092b4 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(vespamalloc_testatomic_app TEST SOURCES testatomic.cpp DEPENDS - atomic + ${VESPA_ATOMIC_LIB} ) vespa_add_test(NAME vespamalloc_testatomic_app NO_VALGRIND COMMAND vespamalloc_testatomic_app) diff --git a/vespamalloc/src/vespamalloc/CMakeLists.txt b/vespamalloc/src/vespamalloc/CMakeLists.txt index ac7fd300828..1fe550ad95a 100644 --- a/vespamalloc/src/vespamalloc/CMakeLists.txt +++ b/vespamalloc/src/vespamalloc/CMakeLists.txt @@ -5,7 +5,7 @@ vespa_add_library(vespamalloc $ INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespamallocd @@ -14,7 +14,7 @@ vespa_add_library(vespamallocd $ INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespamallocdst16 @@ -23,7 +23,7 @@ vespa_add_library(vespamallocdst16 $ INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespamallocdst16_nl @@ -32,7 +32,7 @@ vespa_add_library(vespamallocdst16_nl $ INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespammap -- cgit v1.2.3 From c6f4c3c58e85b059caeb33b6654e713977408b3d Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Sun, 26 Apr 2020 21:43:17 +0200 Subject: Use fewer threads to get next generation of config Check for next config generation in one thread. Simplify Subscriber. Use a ConcurrentHashMap for active subscribers. Reduces the number of threads used by config proxy from ~230 to ~30 --- .../vespa/config/proxy/ConfigSourceClient.java | 2 - .../config/proxy/MemoryCacheConfigClient.java | 3 - .../vespa/config/proxy/RpcConfigSourceClient.java | 91 ++++++++++++---------- .../com/yahoo/vespa/config/proxy/Subscriber.java | 64 ++++++++++++++- .../config/proxy/UpstreamConfigSubscriber.java | 82 ------------------- .../vespa/config/proxy/MockConfigSourceClient.java | 3 - 6 files changed, 111 insertions(+), 134 deletions(-) delete mode 100644 config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java index 9fb78e7e812..e9110a0bc78 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java @@ -24,8 +24,6 @@ interface ConfigSourceClient { List getSourceConnections(); - void updateSubscribers(RawConfig config); - DelayedResponses delayedResponses(); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java index df0b274f32b..2e55b9a6f86 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java @@ -59,9 +59,6 @@ class MemoryCacheConfigClient implements ConfigSourceClient { return Collections.singletonList("N/A"); } - @Override - public void updateSubscribers(RawConfig config) {} - @Override public DelayedResponses delayedResponses() { return delayedResponses; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index 12fd95b8503..ba58c369afd 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -10,6 +10,10 @@ import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import com.yahoo.vespa.config.ConfigCacheKey; import com.yahoo.vespa.config.RawConfig; @@ -17,15 +21,14 @@ import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.concurrent.DelayQueue; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.logging.Logger; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; /** @@ -33,7 +36,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; * * @author hmusum */ -class RpcConfigSourceClient implements ConfigSourceClient { +class RpcConfigSourceClient implements ConfigSourceClient, Runnable { private final static Logger log = Logger.getLogger(RpcConfigSourceClient.class.getName()); private static final double timingValuesRatio = 0.8; @@ -42,16 +45,18 @@ class RpcConfigSourceClient implements ConfigSourceClient { private final RpcServer rpcServer; private final ConfigSourceSet configSourceSet; - private final HashMap activeSubscribers = new HashMap<>(); - private final Object activeSubscribersLock = new Object(); + private final Map activeSubscribers = new ConcurrentHashMap<>(); private final MemoryCache memoryCache; private final DelayedResponses delayedResponses; private final static TimingValues timingValues; - private final ExecutorService exec; + private final ScheduledExecutorService nextConfigScheduler = + Executors.newScheduledThreadPool(1, new DaemonThreadFactory("next config")); + private ScheduledFuture nextConfigFuture; private final JRTConfigRequester requester; // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory()); - private ScheduledFuture delayedResponseScheduler; + private final ScheduledExecutorService delayedResponsesScheduler = + Executors.newScheduledThreadPool(1, new DaemonThreadFactory("delayed responses")); + private ScheduledFuture delayedResponsesFuture; static { // Proxy should time out before clients upon subscription. @@ -69,14 +74,10 @@ class RpcConfigSourceClient implements ConfigSourceClient { this.memoryCache = memoryCache; this.delayedResponses = new DelayedResponses(); checkConfigSources(); - exec = Executors.newCachedThreadPool(new DaemonThreadFactory("subscriber-")); + nextConfigFuture = nextConfigScheduler.scheduleAtFixedRate(this, 0, 10, MILLISECONDS); requester = JRTConfigRequester.create(configSourceSet, timingValues); - // Wait for 5 seconds initially, then run every second - delayedResponseScheduler = scheduler.scheduleAtFixedRate( - new DelayedResponseHandler(delayedResponses, memoryCache, rpcServer), - 5, - 1, - SECONDS); + DelayedResponseHandler command = new DelayedResponseHandler(delayedResponses, memoryCache, rpcServer); + delayedResponsesFuture = delayedResponsesScheduler.scheduleAtFixedRate(command, 5, 1, SECONDS); } /** @@ -153,30 +154,36 @@ class RpcConfigSourceClient implements ConfigSourceClient { } private void subscribeToConfig(RawConfig input, ConfigCacheKey configCacheKey) { - synchronized (activeSubscribersLock) { - if (activeSubscribers.containsKey(configCacheKey)) { - log.log(Level.FINE, () -> "Already a subscriber running for: " + configCacheKey); - } else { - log.log(Level.FINE, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey); - UpstreamConfigSubscriber subscriber = - new UpstreamConfigSubscriber(input, this, configSourceSet, timingValues, requester, memoryCache); - try { - subscriber.subscribe(); - activeSubscribers.put(configCacheKey, subscriber); - exec.execute(subscriber); - } catch (ConfigurationRuntimeException e) { - log.log(Level.INFO, "Subscribe for '" + configCacheKey + "' failed, closing subscriber"); - subscriber.cancel(); - } - } + if (activeSubscribers.containsKey(configCacheKey)) return; + + log.log(Level.FINE, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey); + var subscriber = new Subscriber(input, configSourceSet, timingValues, requester); + try { + subscriber.subscribe(); + activeSubscribers.put(configCacheKey, subscriber); + } catch (ConfigurationRuntimeException e) { + log.log(Level.INFO, "Subscribe for '" + configCacheKey + "' failed, closing subscriber"); + subscriber.cancel(); } } + @Override + public void run() { + activeSubscribers.values().forEach(subscriber -> { + if (!subscriber.isClosed()) { + Optional config = subscriber.nextGeneration(); + config.ifPresent(this::updateWithNewConfig); + } + }); + } + @Override public void cancel() { shutdownSourceConnections(); - delayedResponseScheduler.cancel(true); - scheduler.shutdown(); + delayedResponsesFuture.cancel(true); + delayedResponsesScheduler.shutdown(); + nextConfigFuture.cancel(true); + nextConfigScheduler.shutdown(); } /** @@ -184,13 +191,9 @@ class RpcConfigSourceClient implements ConfigSourceClient { */ @Override public void shutdownSourceConnections() { - synchronized (activeSubscribersLock) { - for (Subscriber subscriber : activeSubscribers.values()) { - subscriber.cancel(); - } - activeSubscribers.clear(); - } - exec.shutdown(); + activeSubscribers.values().forEach(Subscriber::cancel); + activeSubscribers.clear(); + nextConfigScheduler.shutdown(); requester.close(); } @@ -250,4 +253,12 @@ class RpcConfigSourceClient implements ConfigSourceClient { return delayedResponses; } + private void updateWithNewConfig(RawConfig newConfig) { + log.log(Level.FINE, () -> "config to be returned for '" + newConfig.getKey() + + "', generation=" + newConfig.getGeneration() + + ", payload=" + newConfig.getPayload()); + memoryCache.update(newConfig); + updateSubscribers(newConfig); + } + } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java index 74c99b7670b..f96d6470679 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java @@ -1,12 +1,68 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.impl.GenericConfigHandle; +import com.yahoo.config.subscription.impl.GenericConfigSubscriber; +import com.yahoo.config.subscription.impl.JRTConfigRequester; + +import java.util.Optional; +import java.util.logging.Level; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.TimingValues; +import com.yahoo.yolean.Exceptions; + +import java.util.Map; +import java.util.logging.Logger; + /** - * Interface for subscribing to config from upstream config sources. - * * @author hmusum */ -public interface Subscriber extends Runnable { +public class Subscriber { + + private final static Logger log = Logger.getLogger(Subscriber.class.getName()); + + private final RawConfig config; + private final ConfigSourceSet configSourceSet; + private final TimingValues timingValues; + private final GenericConfigSubscriber subscriber; + private GenericConfigHandle handle; + + Subscriber(RawConfig config, ConfigSourceSet configSourceSet, TimingValues timingValues, JRTConfigRequester requester) { + this.config = config; + this.configSourceSet = configSourceSet; + this.timingValues = timingValues; + this.subscriber = new GenericConfigSubscriber(Map.of(configSourceSet, requester)); + } + + void subscribe() { + ConfigKey key = config.getKey(); + handle = subscriber.subscribe(new ConfigKey<>(key.getName(), key.getConfigId(), key.getNamespace()), + config.getDefContent(), configSourceSet, timingValues); + } + + public Optional nextGeneration() { + if (subscriber.nextGeneration(0)) { + try { + return Optional.of(handle.getRawConfig()); + } catch (Exception e) { // To avoid thread throwing exception and loop never running this again + log.log(Level.WARNING, "Got exception: " + Exceptions.toMessageString(e)); + } catch (Throwable e) { + com.yahoo.protect.Process.logAndDie("Got error, exiting: " + Exceptions.toMessageString(e)); + } + } + return Optional.empty(); + } + + public void cancel() { + if (subscriber != null) { + subscriber.close(); + } + } + + boolean isClosed() { + return subscriber.isClosed(); + } - void cancel(); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java deleted file mode 100644 index a24407588be..00000000000 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.proxy; - -import com.yahoo.config.subscription.ConfigSourceSet; -import com.yahoo.config.subscription.impl.GenericConfigHandle; -import com.yahoo.config.subscription.impl.GenericConfigSubscriber; -import com.yahoo.config.subscription.impl.JRTConfigRequester; -import java.util.logging.Level; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.RawConfig; -import com.yahoo.vespa.config.TimingValues; -import com.yahoo.yolean.Exceptions; - -import java.util.Map; -import java.util.logging.Logger; - -/** - * @author hmusum - */ -public class UpstreamConfigSubscriber implements Subscriber { - - private final static Logger log = Logger.getLogger(UpstreamConfigSubscriber.class.getName()); - - private final RawConfig config; - private final ConfigSourceClient configSourceClient; - private final ConfigSourceSet configSourceSet; - private final TimingValues timingValues; - private final JRTConfigRequester requester; - private final MemoryCache memoryCache; - private GenericConfigSubscriber subscriber; - private GenericConfigHandle handle; - - UpstreamConfigSubscriber(RawConfig config, ConfigSourceClient configSourceClient, ConfigSourceSet configSourceSet, - TimingValues timingValues, JRTConfigRequester requester, - MemoryCache memoryCache) { - this.config = config; - this.configSourceClient = configSourceClient; - this.configSourceSet = configSourceSet; - this.timingValues = timingValues; - this.requester = requester; - this.memoryCache = memoryCache; - } - - void subscribe() { - subscriber = new GenericConfigSubscriber(Map.of(configSourceSet, requester)); - ConfigKey key = config.getKey(); - handle = subscriber.subscribe(new ConfigKey<>(key.getName(), key.getConfigId(), key.getNamespace()), - config.getDefContent(), configSourceSet, timingValues); - } - - @Override - public void run() { - do { - if (! subscriber.nextGeneration()) continue; - - try { - updateWithNewConfig(handle); - } catch (Exception e) { // To avoid thread throwing exception and loop never running this again - log.log(Level.WARNING, "Got exception: " + Exceptions.toMessageString(e)); - } catch (Throwable e) { - com.yahoo.protect.Process.logAndDie("Got error, exiting: " + Exceptions.toMessageString(e)); - } - } while (!subscriber.isClosed()); - } - - private void updateWithNewConfig(GenericConfigHandle handle) { - RawConfig newConfig = handle.getRawConfig(); - log.log(Level.FINE, () -> "config to be returned for '" + newConfig.getKey() + - "', generation=" + newConfig.getGeneration() + - ", payload=" + newConfig.getPayload()); - memoryCache.update(newConfig); - configSourceClient.updateSubscribers(newConfig); - } - - @Override - public void cancel() { - if (subscriber != null) { - subscriber.close(); - } - } - -} diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java index 06e55eef4fa..2126d673da0 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java @@ -53,9 +53,6 @@ public class MockConfigSourceClient implements ConfigSourceClient{ return Collections.singletonList("N/A"); } - @Override - public void updateSubscribers(RawConfig config) { } - @Override public DelayedResponses delayedResponses() { return delayedResponses; } -- cgit v1.2.3 From 7ba5b90e295197f986a35870f6aff24ab3f4f3c3 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Sun, 26 Apr 2020 22:15:34 +0200 Subject: Use llvm 8 on RHEL 8. --- default_build_settings.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 66fcf1b2e16..073d0d17620 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -17,7 +17,7 @@ endfunction() function(setup_vespa_default_build_settings_rhel_8_1) message("-- Setting up default build settings for rhel 8.1") - set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_centos_7) -- cgit v1.2.3 From 42a48b1018f06933e9abb8cee700015dcb202e74 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Sun, 26 Apr 2020 22:26:19 +0200 Subject: Adjust compiler options for RHEL 8 to get openblas header files. Use same default build settings setup function for minor RHEL releases. --- default_build_settings.cmake | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 073d0d17620..67d2be5917d 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -8,15 +8,16 @@ function(setup_vespa_default_build_settings_rhel_6_10) set(DEFAULT_CMAKE_SHARED_LINKER_FLAGS "-lrt" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_rhel_7_7) - message("-- Setting up default build settings for rhel 7.7") +function(setup_vespa_default_build_settings_rhel_7) + message("-- Setting up default build settings for rhel 7") set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm7.0/lib" PARENT_SCOPE) set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm7.0" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_rhel_8_1) - message("-- Setting up default build settings for rhel 8.1") +function(setup_vespa_default_build_settings_rhel_8) + message("-- Setting up default build settings for rhel 8") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) endfunction() @@ -159,9 +160,9 @@ function(vespa_use_default_build_settings) if(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 6.10") setup_vespa_default_build_settings_rhel_6_10() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 7.7") - setup_vespa_default_build_settings_rhel_7_7() + setup_vespa_default_build_settings_rhel_7() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.1") - setup_vespa_default_build_settings_rhel_8_1() + setup_vespa_default_build_settings_rhel_8() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 7") setup_vespa_default_build_settings_centos_7() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8") -- cgit v1.2.3 From 4663fd3fab5ee4d7a00c7549a96628d44f27ca6d Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sun, 26 Apr 2020 23:09:09 +0000 Subject: Prepare for making persistence layer async. Avoid state in the thread. --- .../persistence/persistencethread_splittest.cpp | 2 +- storage/src/tests/persistence/testandsettest.cpp | 36 +++--- .../storage/persistence/persistencethread.cpp | 122 +++++++++++---------- .../vespa/storage/persistence/persistencethread.h | 29 +++-- .../vespa/storage/persistence/testandsethelper.cpp | 12 +- .../vespa/storage/persistence/testandsethelper.h | 4 +- 6 files changed, 103 insertions(+), 102 deletions(-) diff --git a/storage/src/tests/persistence/persistencethread_splittest.cpp b/storage/src/tests/persistence/persistencethread_splittest.cpp index ea7dce96e0c..c8318bef211 100644 --- a/storage/src/tests/persistence/persistencethread_splittest.cpp +++ b/storage/src/tests/persistence/persistencethread_splittest.cpp @@ -213,7 +213,7 @@ PersistenceThreadSplitTest::doTest(SplitCase splitCase) cmd.setMinByteSize(maxSize); cmd.setMinDocCount(maxCount); cmd.setSourceIndex(0); - MessageTracker::UP result(thread->handleSplitBucket(cmd)); + MessageTracker::UP result(thread->handleSplitBucket(cmd, context)); api::ReturnCode code(result->getResult()); EXPECT_EQ(error, code); if (!code.success()) { diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index 1507f0e8f0d..1e67e90b540 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -33,15 +33,15 @@ struct TestAndSetTest : SingleDiskPersistenceTestUtils { unique_ptr thread; shared_ptr testDoc; document::DocumentId testDocId; + spi::Context context; + + TestAndSetTest() + : context(spi::LoadType(0, "default"), 0, 0) + {} void SetUp() override { SingleDiskPersistenceTestUtils::SetUp(); - spi::Context context( - spi::LoadType(0, "default"), - spi::Priority(0), - spi::Trace::TraceLevel(0)); - createBucket(BUCKET_ID); getPersistenceProvider().createBucket( makeSpiBucket(BUCKET_ID), @@ -85,7 +85,7 @@ TEST_F(TestAndSetTest, conditional_put_not_executed_on_condition_mismatch) { api::PutCommand putTwo(makeDocumentBucket(BUCKET_ID), testDoc, timestampTwo); setTestCondition(putTwo); - ASSERT_EQ(thread->handlePut(putTwo)->getResult().getResult(), + ASSERT_EQ(thread->handlePut(putTwo, context)->getResult().getResult(), api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); } @@ -105,7 +105,7 @@ TEST_F(TestAndSetTest, conditional_put_executed_on_condition_match) { api::PutCommand putTwo(makeDocumentBucket(BUCKET_ID), testDoc, timestampTwo); setTestCondition(putTwo); - ASSERT_EQ(thread->handlePut(putTwo)->getResult().getResult(), api::ReturnCode::Result::OK); + ASSERT_EQ(thread->handlePut(putTwo, context)->getResult().getResult(), api::ReturnCode::Result::OK); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId) + expectedDocEntryString(timestampTwo, testDocId), dumpBucket(BUCKET_ID)); @@ -125,7 +125,7 @@ TEST_F(TestAndSetTest, conditional_remove_not_executed_on_condition_mismatch) { api::RemoveCommand remove(makeDocumentBucket(BUCKET_ID), testDocId, timestampTwo); setTestCondition(remove); - ASSERT_EQ(thread->handleRemove(remove)->getResult().getResult(), + ASSERT_EQ(thread->handleRemove(remove, context)->getResult().getResult(), api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); @@ -145,7 +145,7 @@ TEST_F(TestAndSetTest, conditional_remove_executed_on_condition_match) { api::RemoveCommand remove(makeDocumentBucket(BUCKET_ID), testDocId, timestampTwo); setTestCondition(remove); - ASSERT_EQ(thread->handleRemove(remove)->getResult().getResult(), api::ReturnCode::Result::OK); + ASSERT_EQ(thread->handleRemove(remove, context)->getResult().getResult(), api::ReturnCode::Result::OK); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId) + expectedDocEntryString(timestampTwo, testDocId, spi::REMOVE_ENTRY), dumpBucket(BUCKET_ID)); @@ -172,7 +172,7 @@ TEST_F(TestAndSetTest, conditional_update_not_executed_on_condition_mismatch) { putTestDocument(false, timestampOne); auto updateUp = conditional_update_test(false, timestampTwo); - ASSERT_EQ(thread->handleUpdate(*updateUp)->getResult().getResult(), + ASSERT_EQ(thread->handleUpdate(*updateUp, context)->getResult().getResult(), api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); @@ -185,7 +185,7 @@ TEST_F(TestAndSetTest, conditional_update_executed_on_condition_match) { putTestDocument(true, timestampOne); auto updateUp = conditional_update_test(false, timestampTwo); - ASSERT_EQ(thread->handleUpdate(*updateUp)->getResult().getResult(), api::ReturnCode::Result::OK); + ASSERT_EQ(thread->handleUpdate(*updateUp, context)->getResult().getResult(), api::ReturnCode::Result::OK); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId) + expectedDocEntryString(timestampTwo, testDocId), dumpBucket(BUCKET_ID)); @@ -197,7 +197,7 @@ TEST_F(TestAndSetTest, conditional_update_not_executed_when_no_document_and_no_a api::Timestamp updateTimestamp = 200; auto updateUp = conditional_update_test(false, updateTimestamp); - ASSERT_EQ(thread->handleUpdate(*updateUp)->getResult().getResult(), + ASSERT_EQ(thread->handleUpdate(*updateUp, context)->getResult().getResult(), api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); EXPECT_EQ("", dumpBucket(BUCKET_ID)); } @@ -206,7 +206,7 @@ TEST_F(TestAndSetTest, conditional_update_executed_when_no_document_but_auto_cre api::Timestamp updateTimestamp = 200; auto updateUp = conditional_update_test(true, updateTimestamp); - ASSERT_EQ(thread->handleUpdate(*updateUp)->getResult().getResult(), api::ReturnCode::Result::OK); + ASSERT_EQ(thread->handleUpdate(*updateUp, context)->getResult().getResult(), api::ReturnCode::Result::OK); EXPECT_EQ(expectedDocEntryString(updateTimestamp, testDocId), dumpBucket(BUCKET_ID)); assertTestDocumentFoundAndMatchesContent(NEW_CONTENT); } @@ -218,7 +218,7 @@ TEST_F(TestAndSetTest, invalid_document_selection_should_fail) { api::PutCommand put(makeDocumentBucket(BUCKET_ID), testDoc, timestamp); put.setCondition(documentapi::TestAndSetCondition("bjarne")); - ASSERT_EQ(thread->handlePut(put)->getResult().getResult(), api::ReturnCode::Result::ILLEGAL_PARAMETERS); + ASSERT_EQ(thread->handlePut(put, context)->getResult().getResult(), api::ReturnCode::Result::ILLEGAL_PARAMETERS); EXPECT_EQ("", dumpBucket(BUCKET_ID)); } @@ -228,9 +228,9 @@ TEST_F(TestAndSetTest, conditional_put_to_non_existing_document_should_fail) { api::Timestamp timestamp = 0; api::PutCommand put(makeDocumentBucket(BUCKET_ID), testDoc, timestamp); setTestCondition(put); - thread->handlePut(put); + thread->handlePut(put, context); - ASSERT_EQ(thread->handlePut(put)->getResult().getResult(), + ASSERT_EQ(thread->handlePut(put, context)->getResult().getResult(), api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); EXPECT_EQ("", dumpBucket(BUCKET_ID)); } @@ -254,7 +254,7 @@ TestAndSetTest::createTestDocument() document::Document::SP TestAndSetTest::retrieveTestDocument() { api::GetCommand get(makeDocumentBucket(BUCKET_ID), testDocId, "[all]"); - auto tracker = thread->handleGet(get); + auto tracker = thread->handleGet(get, context); assert(tracker->getResult() == api::ReturnCode::Result::OK); auto & reply = static_cast(*tracker->getReply()); @@ -274,7 +274,7 @@ void TestAndSetTest::putTestDocument(bool matchingHeader, api::Timestamp timesta } api::PutCommand put(makeDocumentBucket(BUCKET_ID), testDoc, timestamp); - thread->handlePut(put); + thread->handlePut(put, context); } void TestAndSetTest::assertTestDocumentFoundAndMatchesContent(const document::FieldValue & value) diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 4bcd92293d3..60cab4d1216 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -29,7 +29,6 @@ PersistenceThread::PersistenceThread(ServiceLayerComponentRegister& compReg, _processAllHandler(_env, provider), _mergeHandler(_spi, _env), _diskMoveHandler(_env, _spi), - _context(documentapi::LoadType::DEFAULT, 0, 0), _bucketOwnershipNotifier(), _flushMonitor(), _closed(false) @@ -86,11 +85,11 @@ bool PersistenceThread::tasConditionExists(const api::TestAndSetCommand & cmd) { } bool PersistenceThread::tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTracker & tracker, - bool missingDocumentImpliesMatch) { + spi::Context & context, bool missingDocumentImpliesMatch) { try { TestAndSetHelper helper(*this, cmd, missingDocumentImpliesMatch); - auto code = helper.retrieveAndMatch(); + auto code = helper.retrieveAndMatch(context); if (code.failed()) { tracker.fail(code.getResult(), code.getMessage()); return false; @@ -105,35 +104,35 @@ bool PersistenceThread::tasConditionMatches(const api::TestAndSetCommand & cmd, } MessageTracker::UP -PersistenceThread::handlePut(api::PutCommand& cmd) +PersistenceThread::handlePut(api::PutCommand& cmd, spi::Context & context) { auto& metrics = _env._metrics.put[cmd.getLoadType()]; auto tracker = std::make_unique(metrics, _env._component.getClock()); metrics.request_size.addValue(cmd.getApproxByteSize()); - if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker)) { + if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker, context)) { return tracker; } spi::Result response = _spi.put(getBucket(cmd.getDocumentId(), cmd.getBucket()), - spi::Timestamp(cmd.getTimestamp()), cmd.getDocument(), _context); + spi::Timestamp(cmd.getTimestamp()), cmd.getDocument(), context); checkForError(response, *tracker); return tracker; } MessageTracker::UP -PersistenceThread::handleRemove(api::RemoveCommand& cmd) +PersistenceThread::handleRemove(api::RemoveCommand& cmd, spi::Context & context) { auto& metrics = _env._metrics.remove[cmd.getLoadType()]; auto tracker = std::make_unique(metrics,_env._component.getClock()); metrics.request_size.addValue(cmd.getApproxByteSize()); - if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker)) { + if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker, context)) { return tracker; } spi::RemoveResult response = _spi.removeIfFound(getBucket(cmd.getDocumentId(), cmd.getBucket()), - spi::Timestamp(cmd.getTimestamp()), cmd.getDocumentId(), _context); + spi::Timestamp(cmd.getTimestamp()), cmd.getDocumentId(), context); if (checkForError(response, *tracker)) { tracker->setReply(std::make_shared(cmd, response.wasFound() ? cmd.getTimestamp() : 0)); } @@ -144,18 +143,18 @@ PersistenceThread::handleRemove(api::RemoveCommand& cmd) } MessageTracker::UP -PersistenceThread::handleUpdate(api::UpdateCommand& cmd) +PersistenceThread::handleUpdate(api::UpdateCommand& cmd, spi::Context & context) { auto& metrics = _env._metrics.update[cmd.getLoadType()]; auto tracker = std::make_unique(metrics, _env._component.getClock()); metrics.request_size.addValue(cmd.getApproxByteSize()); - if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker, cmd.getUpdate()->getCreateIfNonExistent())) { + if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker, context, cmd.getUpdate()->getCreateIfNonExistent())) { return tracker; } spi::UpdateResult response = _spi.update(getBucket(cmd.getUpdate()->getId(), cmd.getBucket()), - spi::Timestamp(cmd.getTimestamp()), cmd.getUpdate(), _context); + spi::Timestamp(cmd.getTimestamp()), cmd.getUpdate(), context); if (checkForError(response, *tracker)) { auto reply = std::make_shared(cmd); reply->setOldTimestamp(response.getExistingTimestamp()); @@ -177,7 +176,7 @@ spi::ReadConsistency api_read_consistency_to_spi(api::InternalReadConsistency co } MessageTracker::UP -PersistenceThread::handleGet(api::GetCommand& cmd) +PersistenceThread::handleGet(api::GetCommand& cmd, spi::Context & context) { auto& metrics = _env._metrics.get[cmd.getLoadType()]; auto tracker = std::make_unique(metrics,_env._component.getClock()); @@ -186,9 +185,9 @@ PersistenceThread::handleGet(api::GetCommand& cmd) document::FieldSetRepo repo; document::FieldSet::UP fieldSet = repo.parse(*_env._component.getTypeRepo(), cmd.getFieldSet()); // _context is reset per command, so it's safe to modify it like this. - _context.setReadConsistency(api_read_consistency_to_spi(cmd.internal_read_consistency())); + context.setReadConsistency(api_read_consistency_to_spi(cmd.internal_read_consistency())); spi::GetResult result = - _spi.get(getBucket(cmd.getDocumentId(), cmd.getBucket()), *fieldSet, cmd.getDocumentId(), _context); + _spi.get(getBucket(cmd.getDocumentId(), cmd.getBucket()), *fieldSet, cmd.getDocumentId(), context); if (checkForError(result, *tracker)) { if (!result.hasDocument()) { @@ -227,19 +226,19 @@ PersistenceThread::handleRepairBucket(RepairBucketCommand& cmd) } MessageTracker::UP -PersistenceThread::handleRevert(api::RevertCommand& cmd) +PersistenceThread::handleRevert(api::RevertCommand& cmd, spi::Context & context) { auto tracker = std::make_unique(_env._metrics.revert[cmd.getLoadType()],_env._component.getClock()); spi::Bucket b = spi::Bucket(cmd.getBucket(), spi::PartitionId(_env._partition)); const std::vector & tokens = cmd.getRevertTokens(); for (const api::Timestamp & token : tokens) { - spi::Result result = _spi.removeEntry(b, spi::Timestamp(token), _context); + spi::Result result = _spi.removeEntry(b, spi::Timestamp(token), context); } return tracker; } MessageTracker::UP -PersistenceThread::handleCreateBucket(api::CreateBucketCommand& cmd) +PersistenceThread::handleCreateBucket(api::CreateBucketCommand& cmd, spi::Context & context) { auto tracker = std::make_unique(_env._metrics.createBuckets,_env._component.getClock()); LOG(debug, "CreateBucket(%s)", cmd.getBucketId().toString().c_str()); @@ -248,7 +247,7 @@ PersistenceThread::handleCreateBucket(api::CreateBucketCommand& cmd) DUMP_LOGGED_BUCKET_OPERATIONS(cmd.getBucketId()); } spi::Bucket spiBucket(cmd.getBucket(), spi::PartitionId(_env._partition)); - _spi.createBucket(spiBucket, _context); + _spi.createBucket(spiBucket, context); if (cmd.getActive()) { _spi.setActiveState(spiBucket, spi::BucketInfo::ACTIVE); } @@ -297,7 +296,7 @@ PersistenceThread::checkProviderBucketInfoMatches(const spi::Bucket& bucket, con } MessageTracker::UP -PersistenceThread::handleDeleteBucket(api::DeleteBucketCommand& cmd) +PersistenceThread::handleDeleteBucket(api::DeleteBucketCommand& cmd, spi::Context & context) { auto tracker = std::make_unique(_env._metrics.deleteBuckets,_env._component.getClock()); LOG(debug, "DeletingBucket(%s)", cmd.getBucketId().toString().c_str()); @@ -310,7 +309,7 @@ PersistenceThread::handleDeleteBucket(api::DeleteBucketCommand& cmd) if (!checkProviderBucketInfoMatches(bucket, cmd.getBucketInfo())) { return tracker; } - _spi.deleteBucket(bucket, _context); + _spi.deleteBucket(bucket, context); StorBucketDatabase& db(_env.getBucketDatabase(cmd.getBucket().getBucketSpace())); { StorBucketDatabase::WrappedEntry entry(db.get(cmd.getBucketId(), "FileStorThread::onDeleteBucket")); @@ -335,10 +334,10 @@ PersistenceThread::handleDeleteBucket(api::DeleteBucketCommand& cmd) } MessageTracker::UP -PersistenceThread::handleGetIter(GetIterCommand& cmd) +PersistenceThread::handleGetIter(GetIterCommand& cmd, spi::Context & context) { auto tracker = std::make_unique(_env._metrics.visit[cmd.getLoadType()],_env._component.getClock()); - spi::IterateResult result(_spi.iterate(cmd.getIteratorId(), cmd.getMaxByteSize(), _context)); + spi::IterateResult result(_spi.iterate(cmd.getIteratorId(), cmd.getMaxByteSize(), context)); if (checkForError(result, *tracker)) { GetIterReply::SP reply(new GetIterReply(cmd)); reply->getEntries() = result.steal_entries(); @@ -376,16 +375,16 @@ PersistenceThread::handleReadBucketInfo(ReadBucketInfo& cmd) } MessageTracker::UP -PersistenceThread::handleCreateIterator(CreateIteratorCommand& cmd) +PersistenceThread::handleCreateIterator(CreateIteratorCommand& cmd, spi::Context & context) { auto tracker = std::make_unique(_env._metrics.createIterator,_env._component.getClock()); document::FieldSetRepo repo; document::FieldSet::UP fieldSet = repo.parse(*_env._component.getTypeRepo(), cmd.getFields()); // _context is reset per command, so it's safe to modify it like this. - _context.setReadConsistency(cmd.getReadConsistency()); + context.setReadConsistency(cmd.getReadConsistency()); spi::CreateIteratorResult result(_spi.createIterator( spi::Bucket(cmd.getBucket(), spi::PartitionId(_env._partition)), - *fieldSet, cmd.getSelection(), cmd.getIncludedVersions(), _context)); + *fieldSet, cmd.getSelection(), cmd.getIncludedVersions(), context)); if (checkForError(result, *tracker)) { tracker->setReply(std::make_shared(cmd, spi::IteratorId(result.getIteratorId()))); } @@ -393,7 +392,7 @@ PersistenceThread::handleCreateIterator(CreateIteratorCommand& cmd) } MessageTracker::UP -PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd) +PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd, spi::Context & context) { auto tracker = std::make_unique(_env._metrics.splitBuckets,_env._component.getClock()); NotificationGuard notifyGuard(*_bucketOwnershipNotifier); @@ -414,7 +413,7 @@ PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd) SplitBitDetector::Result targetInfo; if (_env._config.enableMultibitSplitOptimalization) { targetInfo = SplitBitDetector::detectSplit(_spi, spiBucket, cmd.getMaxSplitBits(), - _context, cmd.getMinDocCount(), cmd.getMinByteSize()); + context, cmd.getMinDocCount(), cmd.getMinByteSize()); } if (targetInfo.empty() || !_env._config.enableMultibitSplitOptimalization) { document::BucketId src(cmd.getBucketId()); @@ -454,7 +453,7 @@ PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd) } #endif spi::Result result = _spi.split(spiBucket, spi::Bucket(target1, spi::PartitionId(lock1.disk)), - spi::Bucket(target2, spi::PartitionId(lock2.disk)), _context); + spi::Bucket(target2, spi::PartitionId(lock2.disk)), context); if (result.hasError()) { tracker->fail(_env.convertErrorCode(result), result.getErrorMessage()); return tracker; @@ -512,7 +511,7 @@ PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd) spi::PartitionId(targets[i].second.diskIndex))); LOG(debug, "Split target %s was empty, but re-creating it since there are remapped operations queued to it", createTarget.toString().c_str()); - _spi.createBucket(createTarget, _context); + _spi.createBucket(createTarget, context); } splitReply.getSplitInfo().emplace_back(targets[i].second.bucket.getBucketId(), targets[i].first->getBucketInfo()); @@ -557,7 +556,7 @@ PersistenceThread::validateJoinCommand(const api::JoinBucketsCommand& cmd, Messa } MessageTracker::UP -PersistenceThread::handleJoinBuckets(api::JoinBucketsCommand& cmd) +PersistenceThread::handleJoinBuckets(api::JoinBucketsCommand& cmd, spi::Context & context) { auto tracker = std::make_unique(_env._metrics.joinBuckets,_env._component.getClock()); if (!validateJoinCommand(cmd, *tracker)) { @@ -606,11 +605,11 @@ PersistenceThread::handleJoinBuckets(api::JoinBucketsCommand& cmd) _spi.join(spi::Bucket(firstBucket, spi::PartitionId(lock1.disk)), spi::Bucket(secondBucket, spi::PartitionId(lock2.disk)), spi::Bucket(destBucket, spi::PartitionId(_env._partition)), - _context); + context); if (!checkForError(result, *tracker)) { return tracker; } - result = _spi.flush(spi::Bucket(destBucket, spi::PartitionId(_env._partition)), _context); + result = _spi.flush(spi::Bucket(destBucket, spi::PartitionId(_env._partition)), context); if (!checkForError(result, *tracker)) { return tracker; } @@ -672,7 +671,7 @@ PersistenceThread::handleSetBucketState(api::SetBucketStateCommand& cmd) } MessageTracker::UP -PersistenceThread::handleInternalBucketJoin(InternalBucketJoinCommand& cmd) +PersistenceThread::handleInternalBucketJoin(InternalBucketJoinCommand& cmd, spi::Context & context) { auto tracker = std::make_unique(_env._metrics.internalJoin,_env._component.getClock()); document::Bucket destBucket = cmd.getBucket(); @@ -689,7 +688,7 @@ PersistenceThread::handleInternalBucketJoin(InternalBucketJoinCommand& cmd) _spi.join(spi::Bucket(destBucket, spi::PartitionId(cmd.getDiskOfInstanceToJoin())), spi::Bucket(destBucket, spi::PartitionId(cmd.getDiskOfInstanceToJoin())), spi::Bucket(destBucket, spi::PartitionId(cmd.getDiskOfInstanceToKeep())), - _context); + context); if (checkForError(result, *tracker)) { tracker->setReply(std::make_shared(cmd, _env.getBucketInfo(cmd.getBucket()))); } @@ -727,46 +726,46 @@ PersistenceThread::handleRecheckBucketInfo(RecheckBucketInfoCommand& cmd) } MessageTracker::UP -PersistenceThread::handleCommandSplitByType(api::StorageCommand& msg) +PersistenceThread::handleCommandSplitByType(api::StorageCommand& msg, spi::Context & context) { switch (msg.getType().getId()) { case api::MessageType::GET_ID: - return handleGet(static_cast(msg)); + return handleGet(static_cast(msg), context); case api::MessageType::PUT_ID: - return handlePut(static_cast(msg)); + return handlePut(static_cast(msg), context); case api::MessageType::REMOVE_ID: - return handleRemove(static_cast(msg)); + return handleRemove(static_cast(msg), context); case api::MessageType::UPDATE_ID: - return handleUpdate(static_cast(msg)); + return handleUpdate(static_cast(msg), context); case api::MessageType::REVERT_ID: - return handleRevert(static_cast(msg)); + return handleRevert(static_cast(msg), context); case api::MessageType::CREATEBUCKET_ID: - return handleCreateBucket(static_cast(msg)); + return handleCreateBucket(static_cast(msg), context); case api::MessageType::DELETEBUCKET_ID: - return handleDeleteBucket(static_cast(msg)); + return handleDeleteBucket(static_cast(msg), context); case api::MessageType::JOINBUCKETS_ID: - return handleJoinBuckets(static_cast(msg)); + return handleJoinBuckets(static_cast(msg), context); case api::MessageType::SPLITBUCKET_ID: - return handleSplitBucket(static_cast(msg)); + return handleSplitBucket(static_cast(msg), context); // Depends on iterators case api::MessageType::STATBUCKET_ID: - return _processAllHandler.handleStatBucket(static_cast(msg), _context); + return _processAllHandler.handleStatBucket(static_cast(msg), context); case api::MessageType::REMOVELOCATION_ID: - return _processAllHandler.handleRemoveLocation(static_cast(msg), _context); + return _processAllHandler.handleRemoveLocation(static_cast(msg), context); case api::MessageType::MERGEBUCKET_ID: - return _mergeHandler.handleMergeBucket(static_cast(msg), _context); + return _mergeHandler.handleMergeBucket(static_cast(msg), context); case api::MessageType::GETBUCKETDIFF_ID: - return _mergeHandler.handleGetBucketDiff(static_cast(msg), _context); + return _mergeHandler.handleGetBucketDiff(static_cast(msg), context); case api::MessageType::APPLYBUCKETDIFF_ID: - return _mergeHandler.handleApplyBucketDiff(static_cast(msg), _context); + return _mergeHandler.handleApplyBucketDiff(static_cast(msg), context); case api::MessageType::SETBUCKETSTATE_ID: return handleSetBucketState(static_cast(msg)); case api::MessageType::INTERNAL_ID: switch(static_cast(msg).getType()) { case GetIterCommand::ID: - return handleGetIter(static_cast(msg)); + return handleGetIter(static_cast(msg), context); case CreateIteratorCommand::ID: - return handleCreateIterator(static_cast(msg)); + return handleCreateIterator(static_cast(msg), context); case ReadBucketList::ID: return handleReadBucketList(static_cast(msg)); case ReadBucketInfo::ID: @@ -774,9 +773,9 @@ PersistenceThread::handleCommandSplitByType(api::StorageCommand& msg) case RepairBucketCommand::ID: return handleRepairBucket(static_cast(msg)); case BucketDiskMoveCommand::ID: - return _diskMoveHandler.handleBucketDiskMove(static_cast(msg), _context); + return _diskMoveHandler.handleBucketDiskMove(static_cast(msg), context); case InternalBucketJoinCommand::ID: - return handleInternalBucketJoin(static_cast(msg)); + return handleInternalBucketJoin(static_cast(msg), context); case RecheckBucketInfoCommand::ID: return handleRecheckBucketInfo(static_cast(msg)); default: @@ -792,13 +791,13 @@ PersistenceThread::handleCommandSplitByType(api::StorageCommand& msg) MessageTracker::UP PersistenceThread::handleCommand(api::StorageCommand& msg) { - _context = spi::Context(msg.getLoadType(), msg.getPriority(), msg.getTrace().getLevel()); - MessageTracker::UP mtracker(handleCommandSplitByType(msg)); - if (mtracker && ! _context.getTrace().getRoot().isEmpty()) { + spi::Context context(msg.getLoadType(), msg.getPriority(), msg.getTrace().getLevel()); + MessageTracker::UP mtracker(handleCommandSplitByType(msg, context)); + if (mtracker && ! context.getTrace().getRoot().isEmpty()) { if (mtracker->getReply()) { - mtracker->getReply()->getTrace().getRoot().addChild(_context.getTrace().getRoot()); + mtracker->getReply()->getTrace().getRoot().addChild(context.getTrace().getRoot()); } else { - msg.getTrace().getRoot().addChild(_context.getTrace().getRoot()); + msg.getTrace().getRoot().addChild(context.getTrace().getRoot()); } } return mtracker; @@ -934,7 +933,10 @@ PersistenceThread::flushAllReplies( } #endif spi::Bucket b(bucket, spi::PartitionId(_env._partition)); - spi::Result result = _spi.flush(b, _context); + // Flush is not used for anything currentlu, and the context is not correct either when batching is done + //So just faking it here. + spi::Context dummyContext(documentapi::LoadType::DEFAULT, 0, 0); + spi::Result result = _spi.flush(b, dummyContext); uint32_t errorCode = _env.convertErrorCode(result); if (errorCode != 0) { for (uint32_t i = 0; i < replies.size(); ++i) { diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index ed27a759e8b..e410843c1be 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -28,21 +28,21 @@ public: void flush() override; framework::Thread& getThread() override { return *_thread; } - MessageTracker::UP handlePut(api::PutCommand& cmd); - MessageTracker::UP handleRemove(api::RemoveCommand& cmd); - MessageTracker::UP handleUpdate(api::UpdateCommand& cmd); - MessageTracker::UP handleGet(api::GetCommand& cmd); - MessageTracker::UP handleRevert(api::RevertCommand& cmd); - MessageTracker::UP handleCreateBucket(api::CreateBucketCommand& cmd); - MessageTracker::UP handleDeleteBucket(api::DeleteBucketCommand& cmd); - MessageTracker::UP handleCreateIterator(CreateIteratorCommand& cmd); - MessageTracker::UP handleGetIter(GetIterCommand& cmd); + MessageTracker::UP handlePut(api::PutCommand& cmd, spi::Context & context); + MessageTracker::UP handleRemove(api::RemoveCommand& cmd, spi::Context & context); + MessageTracker::UP handleUpdate(api::UpdateCommand& cmd, spi::Context & context); + MessageTracker::UP handleGet(api::GetCommand& cmd, spi::Context & context); + MessageTracker::UP handleRevert(api::RevertCommand& cmd, spi::Context & context); + MessageTracker::UP handleCreateBucket(api::CreateBucketCommand& cmd, spi::Context & context); + MessageTracker::UP handleDeleteBucket(api::DeleteBucketCommand& cmd, spi::Context & context); + MessageTracker::UP handleCreateIterator(CreateIteratorCommand& cmd, spi::Context & context); + MessageTracker::UP handleGetIter(GetIterCommand& cmd, spi::Context & context); MessageTracker::UP handleReadBucketList(ReadBucketList& cmd); MessageTracker::UP handleReadBucketInfo(ReadBucketInfo& cmd); - MessageTracker::UP handleJoinBuckets(api::JoinBucketsCommand& cmd); + MessageTracker::UP handleJoinBuckets(api::JoinBucketsCommand& cmd, spi::Context & context); MessageTracker::UP handleSetBucketState(api::SetBucketStateCommand& cmd); - MessageTracker::UP handleInternalBucketJoin(InternalBucketJoinCommand& cmd); - MessageTracker::UP handleSplitBucket(api::SplitBucketCommand& cmd); + MessageTracker::UP handleInternalBucketJoin(InternalBucketJoinCommand& cmd, spi::Context & context); + MessageTracker::UP handleSplitBucket(api::SplitBucketCommand& cmd, spi::Context & context); MessageTracker::UP handleRepairBucket(RepairBucketCommand& cmd); MessageTracker::UP handleRecheckBucketInfo(RecheckBucketInfoCommand& cmd); @@ -56,7 +56,6 @@ private: DiskMoveOperationHandler _diskMoveHandler; ServiceLayerComponent::UP _component; framework::Thread::UP _thread; - spi::Context _context; std::unique_ptr _bucketOwnershipNotifier; vespalib::Monitor _flushMonitor; bool _closed; @@ -72,7 +71,7 @@ private: // Message handling functions MessageTracker::UP handleCommand(api::StorageCommand&); - MessageTracker::UP handleCommandSplitByType(api::StorageCommand&); + MessageTracker::UP handleCommandSplitByType(api::StorageCommand&, spi::Context & context); void handleReply(api::StorageReply&); MessageTracker::UP processMessage(api::StorageMessage& msg); @@ -88,7 +87,7 @@ private: friend class TestAndSetHelper; bool tasConditionExists(const api::TestAndSetCommand & cmd); bool tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTracker & tracker, - bool missingDocumentImpliesMatch = false); + spi::Context & context, bool missingDocumentImpliesMatch = false); }; } // storage diff --git a/storage/src/vespa/storage/persistence/testandsethelper.cpp b/storage/src/vespa/storage/persistence/testandsethelper.cpp index 511d44ad331..e1909252c8f 100644 --- a/storage/src/vespa/storage/persistence/testandsethelper.cpp +++ b/storage/src/vespa/storage/persistence/testandsethelper.cpp @@ -31,12 +31,12 @@ void TestAndSetHelper::parseDocumentSelection() { } } -spi::GetResult TestAndSetHelper::retrieveDocument(const document::FieldSet & fieldSet) { +spi::GetResult TestAndSetHelper::retrieveDocument(const document::FieldSet & fieldSet, spi::Context & context) { return _thread._spi.get( _thread.getBucket(_docId, _cmd.getBucket()), fieldSet, _cmd.getDocumentId(), - _thread._context); + context); } TestAndSetHelper::TestAndSetHelper(PersistenceThread & thread, const api::TestAndSetCommand & cmd, @@ -51,16 +51,16 @@ TestAndSetHelper::TestAndSetHelper(PersistenceThread & thread, const api::TestAn parseDocumentSelection(); } -TestAndSetHelper::~TestAndSetHelper() { -} +TestAndSetHelper::~TestAndSetHelper() = default; -api::ReturnCode TestAndSetHelper::retrieveAndMatch() { +api::ReturnCode +TestAndSetHelper::retrieveAndMatch(spi::Context & context) { // Walk document selection tree to build a minimal field set FieldVisitor fieldVisitor(*_docTypePtr); _docSelectionUp->visit(fieldVisitor); // Retrieve document - auto result = retrieveDocument(fieldVisitor.getFieldSet()); + auto result = retrieveDocument(fieldVisitor.getFieldSet(), context); // If document exists, match it with selection if (result.hasDocument()) { diff --git a/storage/src/vespa/storage/persistence/testandsethelper.h b/storage/src/vespa/storage/persistence/testandsethelper.h index 21c111c712f..b5fa29d0106 100644 --- a/storage/src/vespa/storage/persistence/testandsethelper.h +++ b/storage/src/vespa/storage/persistence/testandsethelper.h @@ -30,13 +30,13 @@ class TestAndSetHelper { void getDocumentType(); void parseDocumentSelection(); - spi::GetResult retrieveDocument(const document::FieldSet & fieldSet); + spi::GetResult retrieveDocument(const document::FieldSet & fieldSet, spi::Context & context); public: TestAndSetHelper(PersistenceThread & thread, const api::TestAndSetCommand & cmd, bool missingDocumentImpliesMatch = false); ~TestAndSetHelper(); - api::ReturnCode retrieveAndMatch(); + api::ReturnCode retrieveAndMatch(spi::Context & context); }; } // storage -- cgit v1.2.3 From 525b875646f6f33580df13b143036d7e05d8052d Mon Sep 17 00:00:00 2001 From: gjoranv Date: Mon, 27 Apr 2020 01:49:38 +0200 Subject: Remove unnecessary dependencies to vespalog. --- application-preprocessor/pom.xml | 5 ----- athenz-identity-provider-service/pom.xml | 6 ------ clustercontroller-utils/pom.xml | 2 +- config-application-package/pom.xml | 6 ------ config-model/pom.xml | 6 ------ container-search-and-docproc/pom.xml | 6 ------ filedistribution/pom.xml | 5 ----- flags/pom.xml | 2 +- metrics-proxy/pom.xml | 6 ------ vespa-maven-plugin/pom.xml | 5 ----- zkfacade/pom.xml | 6 ------ 11 files changed, 2 insertions(+), 53 deletions(-) diff --git a/application-preprocessor/pom.xml b/application-preprocessor/pom.xml index b3b85a776fe..613a2747c35 100644 --- a/application-preprocessor/pom.xml +++ b/application-preprocessor/pom.xml @@ -58,11 +58,6 @@ yolean ${project.version} - - com.yahoo.vespa - vespalog - ${project.version} - junit junit diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index f15ebe87b03..daedc1d0b9b 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -38,12 +38,6 @@ ${project.version} provided - - com.yahoo.vespa - vespalog - ${project.version} - provided - com.fasterxml.jackson.core jackson-core diff --git a/clustercontroller-utils/pom.xml b/clustercontroller-utils/pom.xml index 0fed0c67eb5..bdf57a9caf0 100644 --- a/clustercontroller-utils/pom.xml +++ b/clustercontroller-utils/pom.xml @@ -30,7 +30,7 @@ com.yahoo.vespa - vespalog + vespajlib ${project.version} provided diff --git a/config-application-package/pom.xml b/config-application-package/pom.xml index 2cce4c01243..9deeeb64c8e 100644 --- a/config-application-package/pom.xml +++ b/config-application-package/pom.xml @@ -49,12 +49,6 @@ ${project.version} provided - - com.yahoo.vespa - vespalog - ${project.version} - provided - com.thaiopensource jing diff --git a/config-model/pom.xml b/config-model/pom.xml index 33f6657561c..897236f78e7 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -189,12 +189,6 @@ ${project.version} provided - - com.yahoo.vespa - vespalog - ${project.version} - provided - com.yahoo.vespa statistics diff --git a/container-search-and-docproc/pom.xml b/container-search-and-docproc/pom.xml index 91202087df2..cc66bd66922 100644 --- a/container-search-and-docproc/pom.xml +++ b/container-search-and-docproc/pom.xml @@ -146,12 +146,6 @@ ${project.version} provided - - com.yahoo.vespa - vespalog - ${project.version} - provided - com.yahoo.vespa documentapi diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 133268d5872..8bfa85d8b16 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -28,11 +28,6 @@ jrt ${project.version} - - com.yahoo.vespa - vespalog - ${project.version} - com.yahoo.vespa config diff --git a/flags/pom.xml b/flags/pom.xml index 87fbc30d92d..ba0e4a94692 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -45,7 +45,7 @@ com.yahoo.vespa - vespalog + vespajlib ${project.version} provided diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index 355f420c2a4..8bf5a30e584 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -95,12 +95,6 @@ ${project.version} provided - - com.yahoo.vespa - vespalog - ${project.version} - provided - com.yahoo.vespa yolean diff --git a/vespa-maven-plugin/pom.xml b/vespa-maven-plugin/pom.xml index de3e2f44a67..9f1d6f5ff6b 100644 --- a/vespa-maven-plugin/pom.xml +++ b/vespa-maven-plugin/pom.xml @@ -39,11 +39,6 @@ vespajlib ${project.version} - - com.yahoo.vespa - vespalog - ${project.version} - com.yahoo.vespa config-application-package diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index b857c31d262..de542b89b67 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -71,12 +71,6 @@ slf4j-log4j12 1.7.5 - - com.yahoo.vespa - vespalog - ${project.version} - provided - org.apache.zookeeper zookeeper -- cgit v1.2.3 From dc92064fec43ee5e60df086061c49149a398383c Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Mon, 27 Apr 2020 09:49:52 +0200 Subject: Fix typos in test descriptions. --- .../src/tests/attribute/tensorattribute/tensorattribute_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index a008bc73e93..79fba27bb59 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -739,7 +739,7 @@ TEST_F("onLoads() ignores saved nearest neighbor index if not enabled in config" EXPECT_EQUAL(f.as_dense_tensor().nearest_neighbor_index(), nullptr); } -TEST_F("onLoad() ignores saved nearest neightbor index if major index parameters are changed", DenseTensorAttributeMockIndex) +TEST_F("onLoad() ignores saved nearest neighbor index if major index parameters are changed", DenseTensorAttributeMockIndex) { f.save_example_tensors_with_mock_index(); f.set_hnsw_index_params(HnswIndexParams(5, 20, DistanceMetric::Euclidean)); @@ -750,7 +750,7 @@ TEST_F("onLoad() ignores saved nearest neightbor index if major index parameters index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); } -TEST_F("onLoad() uses saved nearest neightbor index if only minor index parameters are changed", DenseTensorAttributeMockIndex) +TEST_F("onLoad() uses saved nearest neighbor index if only minor index parameters are changed", DenseTensorAttributeMockIndex) { f.save_example_tensors_with_mock_index(); f.set_hnsw_index_params(HnswIndexParams(4, 21, DistanceMetric::Euclidean)); -- cgit v1.2.3 From 391580eefbecc3a97a8527a79a02e8687c8db46a Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 27 Apr 2020 08:49:33 +0000 Subject: Update comments to reflect reality. --- storage/src/vespa/storage/persistence/persistencethread.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 60cab4d1216..e7123164659 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -184,7 +184,6 @@ PersistenceThread::handleGet(api::GetCommand& cmd, spi::Context & context) document::FieldSetRepo repo; document::FieldSet::UP fieldSet = repo.parse(*_env._component.getTypeRepo(), cmd.getFieldSet()); - // _context is reset per command, so it's safe to modify it like this. context.setReadConsistency(api_read_consistency_to_spi(cmd.internal_read_consistency())); spi::GetResult result = _spi.get(getBucket(cmd.getDocumentId(), cmd.getBucket()), *fieldSet, cmd.getDocumentId(), context); @@ -380,7 +379,6 @@ PersistenceThread::handleCreateIterator(CreateIteratorCommand& cmd, spi::Context auto tracker = std::make_unique(_env._metrics.createIterator,_env._component.getClock()); document::FieldSetRepo repo; document::FieldSet::UP fieldSet = repo.parse(*_env._component.getTypeRepo(), cmd.getFields()); - // _context is reset per command, so it's safe to modify it like this. context.setReadConsistency(cmd.getReadConsistency()); spi::CreateIteratorResult result(_spi.createIterator( spi::Bucket(cmd.getBucket(), spi::PartitionId(_env._partition)), @@ -934,7 +932,7 @@ PersistenceThread::flushAllReplies( #endif spi::Bucket b(bucket, spi::PartitionId(_env._partition)); // Flush is not used for anything currentlu, and the context is not correct either when batching is done - //So just faking it here. + // So just faking it here. spi::Context dummyContext(documentapi::LoadType::DEFAULT, 0, 0); spi::Result result = _spi.flush(b, dummyContext); uint32_t errorCode = _env.convertErrorCode(result); -- cgit v1.2.3 From 9295b30afe93b69d7e9b7df421e506c63b0bb376 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 27 Apr 2020 11:21:16 +0200 Subject: Support forgetting nodes completely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Normally we want to remember removed nodes by putting them in the deprovisio´ned state. But if we are sure they will never re-enter the system we should forget them completetely. With this that is achieved by issuing another DELETE on the node when it is in the deprovisioned state. --- .../vespa/hosted/provision/NodeRepository.java | 10 +++++++++- .../provision/restapi/NodesV2ApiHandler.java | 23 +++++++++++++++++----- .../hosted/provision/restapi/NodesV2ApiTest.java | 5 +++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index a4e5255ad09..7045a47acf3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -82,7 +82,8 @@ import java.util.stream.Stream; // 1) (new) | deprovisioned - > provisioned -> (dirty ->) ready -> reserved -> active -> inactive -> dirty -> ready // 2) inactive -> reserved | parked // 3) reserved -> dirty -// 3) * -> failed | parked -> dirty | active | deprovisioned +// 4) * -> failed | parked -> dirty | active | deprovisioned +// 5) deprovisioned -> (forgotten) // Nodes have an application assigned when in states reserved, active and inactive. // Nodes might have an application assigned in dirty. public class NodeRepository extends AbstractComponent { @@ -644,6 +645,13 @@ public class NodeRepository extends AbstractComponent { } } + /** Forgets a deprovisioned node. This removes all traces of the node in the node repository. */ + public void forget(Node node) { + if (node.state() != State.deprovisioned) + throw new IllegalArgumentException(node + " must be deprovisioned before it can be forgotten"); + db.removeNodes(List.of(node)); + } + /** * Throws if the given node cannot be removed. Removal is allowed if: * - Tenant node: node is unallocated diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 1e8bf622f72..b46a940d8ff 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -25,6 +25,7 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -197,17 +198,29 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { private HttpResponse handleDELETE(HttpRequest request) { Path path = new Path(request.getUri()); - if (path.matches("/nodes/v2/node/{hostname}")) { - String hostname = path.get("hostname"); - List removedNodes = nodeRepository.removeRecursively(hostname); - return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", "))); - } + if (path.matches("/nodes/v2/node/{hostname}")) return deleteNode(path.get("hostname")); if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), true); if (path.matches("/nodes/v2/upgrade/firmware")) return cancelFirmwareCheckResponse(); throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); } + private HttpResponse deleteNode(String hostname) { + Optional node = nodeRepository.getNode(hostname); + if (node.isEmpty()) throw new NotFoundException("No node with hostname '" + hostname + "'"); + try (Mutex lock = nodeRepository.lock(node.get())) { + node = nodeRepository.getNode(hostname); + if (node.isEmpty()) throw new NotFoundException("No node with hostname '" + hostname + "'"); + if (node.get().state() == Node.State.deprovisioned) { + nodeRepository.forget(node.get()); + return new MessageResponse("Permanently removed '" + hostname + "'"); + } else { + List removedNodes = nodeRepository.removeRecursively(hostname); + return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", "))); + } + } + } + private Node nodeFromRequest(HttpRequest request) { String hostname = lastElement(request.getUri().getPath()); return nodeRepository.getNode(hostname).orElseThrow(() -> diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 3e23b424675..e7f6b315e8d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -154,6 +154,11 @@ public class NodesV2ApiTest { new byte[0], Request.Method.DELETE), "{\"message\":\"Removed host8.yahoo.com\"}"); + // ... and then forget it completely + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Permanently removed host8.yahoo.com\"}"); + // or, PUT a node in failed ... assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/test-node-pool-102-2", new byte[0], Request.Method.PUT), -- cgit v1.2.3 From ad5de0be92dff181fd8ba9777284825467a36f6f Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 27 Apr 2020 11:31:22 +0200 Subject: Test forgetting on docker host --- .../hosted/provision/restapi/NodesV2ApiHandler.java | 2 +- .../vespa/hosted/provision/restapi/NodesV2ApiTest.java | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index b46a940d8ff..75b1efb649e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -213,7 +213,7 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { if (node.isEmpty()) throw new NotFoundException("No node with hostname '" + hostname + "'"); if (node.get().state() == Node.State.deprovisioned) { nodeRepository.forget(node.get()); - return new MessageResponse("Permanently removed '" + hostname + "'"); + return new MessageResponse("Permanently removed " + hostname); } else { List removedNodes = nodeRepository.removeRecursively(hostname); return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", "))); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index e7f6b315e8d..c16a3f3b3e0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -154,11 +154,6 @@ public class NodesV2ApiTest { new byte[0], Request.Method.DELETE), "{\"message\":\"Removed host8.yahoo.com\"}"); - // ... and then forget it completely - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com", - new byte[0], Request.Method.DELETE), - "{\"message\":\"Permanently removed host8.yahoo.com\"}"); - // or, PUT a node in failed ... assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/test-node-pool-102-2", new byte[0], Request.Method.PUT), @@ -178,7 +173,7 @@ public class NodesV2ApiTest { tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", new byte[0], Request.Method.GET), 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'test-node-pool-102-2'\"}"); - // Put a host in failed and make sure it's children are also failed + // Put a host in failed and make sure its children are also failed assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/dockerhost1.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved dockerhost1.yahoo.com, host4.yahoo.com to failed\"}"); @@ -232,6 +227,17 @@ public class NodesV2ApiTest { .suspend(new HostName("host4.yahoo.com")); assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); + + // move the docker host to deprovisioned + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Removed dockerhost1.yahoo.com\"}"); + // ... and then forget it completely + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Permanently removed dockerhost1.yahoo.com\"}"); + + } @Test -- cgit v1.2.3 From a8467b2d5dd932512219d8eeee9fd73e8a2b267b Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Mon, 27 Apr 2020 12:06:15 +0200 Subject: Create policies for billing API resources --- .../hosted/controller/api/role/PathGroup.java | 17 +++++++- .../vespa/hosted/controller/api/role/Policy.java | 27 ++++++++++++- .../hosted/controller/api/role/RoleDefinition.java | 17 ++++++-- .../vespa/hosted/controller/api/role/RoleTest.java | 45 ++++++++++++++++++++++ 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 5c11dfc2a55..66b06949793 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -58,13 +58,26 @@ enum PathGroup { "/application/v4/tenant/{tenant}/application/", "/application/v4/tenant/{tenant}/cost", "/application/v4/tenant/{tenant}/cost/{date}", - "/routing/v1/status/tenant/{tenant}/{*}", - "/billing/v1/tenant/{tenant}/{*}"), + "/routing/v1/status/tenant/{tenant}/{*}"), tenantKeys(Matcher.tenant, PathPrefix.api, "/application/v4/tenant/{tenant}/key/"), + + billingToken(Matcher.tenant, + PathPrefix.api, + "/billing/v1/tenant/{tenant}/token"), + + billingInstrument(Matcher.tenant, + PathPrefix.api, + "/billing/v1/tenant/{tenant}/instrument", + "/billing/v1/tenant/{tenant}/instrument/{*}"), + + billingList(Matcher.tenant, + PathPrefix.api, + "/billing/v1/tenant/{tenant}/billing/{*}"), + applicationKeys(Matcher.tenant, Matcher.application, PathPrefix.api, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index cfe8d247e54..0afa0668a00 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -142,7 +142,32 @@ enum Policy { /** Access to /payment/notification */ paymentProcessor(Privilege.grant(Action.create) .on(PathGroup.paymentProcessor) - .in(SystemName.PublicCd)); + .in(SystemName.PublicCd)), + + /** Read your own instrument information */ + paymentInstrumentRead(Privilege.grant(Action.read) + .on(PathGroup.billingInstrument) + .in(SystemName.PublicCd)), + + /** Ability to update tenant payment instrument */ + paymentInstrumentUpdate(Privilege.grant(Action.update) + .on(PathGroup.billingInstrument) + .in(SystemName.PublicCd)), + + /** Ability to remove your own payment instrument */ + paymentInstrumentDelete(Privilege.grant(Action.delete) + .on(PathGroup.billingInstrument) + .in(SystemName.PublicCd)), + + /** Get the token to view instrument form */ + paymentInstrumentCreate(Privilege.grant(Action.read) + .on(PathGroup.billingToken) + .in(SystemName.PublicCd)), + + /** Read the generated bills */ + billingInformationRead(Privilege.grant(Action.read) + .on(PathGroup.billingList) + .in(SystemName.PublicCd)); private final Set privileges; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index c05936ee593..438e79bcc4f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -43,7 +43,10 @@ public enum RoleDefinition { reader(Policy.tenantRead, Policy.applicationRead, Policy.deploymentRead, - Policy.publicRead), + Policy.publicRead, + Policy.paymentInstrumentRead, + Policy.paymentInstrumentDelete, + Policy.billingInformationRead), /** User — the dev.ops. role for normal Vespa tenant users */ developer(Policy.applicationCreate, @@ -52,12 +55,20 @@ public enum RoleDefinition { Policy.applicationOperations, Policy.developmentDeployment, Policy.keyManagement, - Policy.submission), + Policy.submission, + Policy.paymentInstrumentRead, + Policy.paymentInstrumentDelete, + Policy.billingInformationRead), /** Admin — the administrative function for user management etc. */ administrator(Policy.tenantUpdate, Policy.tenantManager, - Policy.applicationManager), + Policy.applicationManager, + Policy.paymentInstrumentRead, + Policy.paymentInstrumentUpdate, + Policy.paymentInstrumentDelete, + Policy.paymentInstrumentCreate, + Policy.billingInformationRead), /** Headless — the application specific role identified by deployment keys for production */ headless(Policy.submission), diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index 57b4af9d16c..2da93c5ceca 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -6,8 +6,10 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import org.junit.Test; +import java.awt.event.AdjustmentEvent; import java.net.URI; import java.util.List; +import java.util.stream.Stream; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -19,6 +21,7 @@ public class RoleTest { private static final Enforcer mainEnforcer = new Enforcer(SystemName.main); private static final Enforcer publicEnforcer = new Enforcer(SystemName.Public); + private static final Enforcer publicCdEnforcer = new Enforcer(SystemName.PublicCd); @Test public void operator_membership() { @@ -143,4 +146,46 @@ public class RoleTest { } } + @Test + public void payment_instrument() { + URI paymentInstrumentUri = URI.create("/billing/v1/tenant/t1/instrument/foobar"); + URI tenantPaymentInstrumentUri = URI.create("/billing/v1/tenant/t1/instrument"); + URI tokenUri = URI.create("/billing/v1/tenant/t1/token"); + + Role user = Role.reader(TenantName.from("t1")); + assertTrue(publicCdEnforcer.allows(user, Action.read, paymentInstrumentUri)); + assertTrue(publicCdEnforcer.allows(user, Action.delete, paymentInstrumentUri)); + assertFalse(publicCdEnforcer.allows(user, Action.update, tenantPaymentInstrumentUri)); + assertFalse(publicCdEnforcer.allows(user, Action.read, tokenUri)); + + Role developer = Role.developer(TenantName.from("t1")); + assertTrue(publicCdEnforcer.allows(developer, Action.read, paymentInstrumentUri)); + assertTrue(publicCdEnforcer.allows(developer, Action.delete, paymentInstrumentUri)); + assertFalse(publicCdEnforcer.allows(developer, Action.update, tenantPaymentInstrumentUri)); + assertFalse(publicCdEnforcer.allows(developer, Action.read, tokenUri)); + + Role admin = Role.administrator(TenantName.from("t1")); + assertTrue(publicCdEnforcer.allows(admin, Action.read, paymentInstrumentUri)); + assertTrue(publicCdEnforcer.allows(admin, Action.delete, paymentInstrumentUri)); + assertTrue(publicCdEnforcer.allows(admin, Action.update, tenantPaymentInstrumentUri)); + assertTrue(publicCdEnforcer.allows(admin, Action.read, tokenUri)); + } + + @Test + public void billing() { + URI billing = URI.create("/billing/v1/tenant/t1/billing"); + + Role user = Role.reader(TenantName.from("t1")); + Role developer = Role.developer(TenantName.from("t1")); + Role admin = Role.administrator(TenantName.from("t1")); + + Stream.of(user, developer, admin).forEach(role -> { + assertTrue(publicCdEnforcer.allows(role, Action.read, billing)); + assertFalse(publicCdEnforcer.allows(role, Action.update, billing)); + assertFalse(publicCdEnforcer.allows(role, Action.delete, billing)); + assertFalse(publicCdEnforcer.allows(role, Action.create, billing)); + }); + + } + } -- cgit v1.2.3 From e9c6a3bd0fbceb50f70b64bf4ca77b071598efa5 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Wed, 22 Apr 2020 10:42:50 +0200 Subject: Remove 'nginx-upstream-proxy-protocol' feature flag --- configdefinitions/src/vespa/lb-services.def | 3 --- .../com/yahoo/vespa/config/server/model/LbServicesProducer.java | 5 ----- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 6 ------ 3 files changed, 14 deletions(-) diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index cc496a99c20..33c568061fe 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -4,9 +4,6 @@ namespace=cloud.config -# Enable proxy-protocol for nginx upstreams -nginxUpstreamProxyProtocol bool default=false - # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index 4fe2ec129d3..6366576e163 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -9,9 +9,7 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import java.util.Collections; import java.util.Comparator; @@ -34,17 +32,14 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map> models; private final Zone zone; - private final BooleanFlag nginxUpstreamProxyProtocol; public LbServicesProducer(Map> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; - this.nginxUpstreamProxyProtocol = Flags.NGINX_UPSTREAM_PROXY_PROTOCOL.bindTo(flagSource); } @Override public void getConfig(LbServicesConfig.Builder builder) { - builder.nginxUpstreamProxyProtocol(nginxUpstreamProxyProtocol.value()); models.keySet().stream() .sorted() .forEach(tenant -> { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 85d841e4bb0..22b5ce419d6 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -247,12 +247,6 @@ public class Flags { "Takes effect immediately", APPLICATION_ID); - public static final UnboundBooleanFlag NGINX_UPSTREAM_PROXY_PROTOCOL = defineFeatureFlag( - "nginx-upstream-proxy-protocol", false, - "Whether the nginx should enable proxy-protocol for all upstreams", - "Takes effect immediately", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag NLB_PROXY_PROTOCOL = defineFeatureFlag( "nlb-proxy-protocol", false, "Configure NLB to use proxy protocol", -- cgit v1.2.3 From dfc3ea8a384311135580fc856fde3808640ffeae Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Mon, 27 Apr 2020 14:26:36 +0200 Subject: Fix conflict in path specs --- .../main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java | 1 - 1 file changed, 1 deletion(-) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 66b06949793..ced3d201f6f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -71,7 +71,6 @@ enum PathGroup { billingInstrument(Matcher.tenant, PathPrefix.api, - "/billing/v1/tenant/{tenant}/instrument", "/billing/v1/tenant/{tenant}/instrument/{*}"), billingList(Matcher.tenant, -- cgit v1.2.3 From f0a7a14d639059f4d9de3001fc96d95219f499b2 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 27 Apr 2020 12:28:48 +0000 Subject: Remove batching of messages that has no effect in favor of making async operations easier to implement. --- .../filestorage/filestormanagertest.cpp | 70 -------------- .../src/tests/persistence/persistencequeuetest.cpp | 15 --- .../persistence/filestorage/filestorhandler.cpp | 6 -- .../persistence/filestorage/filestorhandler.h | 5 - .../filestorage/filestorhandlerimpl.cpp | 62 ------------ .../persistence/filestorage/filestorhandlerimpl.h | 6 -- .../storage/persistence/persistencethread.cpp | 107 ++------------------- .../vespa/storage/persistence/persistencethread.h | 2 - 8 files changed, 8 insertions(+), 265 deletions(-) diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 4576f8a08f8..2cb390fca86 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -698,19 +698,6 @@ TEST_F(FileStorManagerTest, handler_pause) { ASSERT_EQ(30, filestorHandler.getNextMessage(0, stripeId).second->getPriority()); } -namespace { - -uint64_t getPutTime(api::StorageMessage::SP& msg) -{ - if (!msg.get()) { - return (uint64_t)-1; - } - - return static_cast(msg.get())->getTimestamp(); -}; - -} - TEST_F(FileStorManagerTest, remap_split) { // Setup a filestorthread to test DummyStorageLink top; @@ -768,63 +755,6 @@ TEST_F(FileStorManagerTest, remap_split) { filestorHandler.dumpQueue(0)); } -TEST_F(FileStorManagerTest, handler_multi) { - // Setup a filestorthread to test - DummyStorageLink top; - DummyStorageLink *dummyManager; - top.push_back(std::unique_ptr( - dummyManager = new DummyStorageLink)); - top.open(); - ForwardingMessageSender messageSender(*dummyManager); - // Since we fake time with small numbers, we need to make sure we dont - // compact them away, as they will seem to be from 1970 - - documentapi::LoadTypeSet loadTypes("raw:"); - FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); - metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1, 1); - - FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), _node->getComponentRegister()); - filestorHandler.setGetNextMessageTimeout(50); - uint32_t stripeId = filestorHandler.getNextStripeId(0); - - std::string content("Here is some content which is in all documents"); - - Document::SP doc1(createDocument(content, "id:footype:testdoctype1:n=1234:bar").release()); - - Document::SP doc2(createDocument(content, "id:footype:testdoctype1:n=4567:bar").release()); - - document::BucketIdFactory factory; - document::BucketId bucket1(16, factory.getBucketId(doc1->getId()).getRawId()); - document::BucketId bucket2(16, factory.getBucketId(doc2->getId()).getRawId()); - - // Populate bucket with the given data - for (uint32_t i = 1; i < 10; i++) { - filestorHandler.schedule( - api::StorageMessage::SP(new api::PutCommand(makeDocumentBucket(bucket1), doc1, i)), 0); - filestorHandler.schedule( - api::StorageMessage::SP(new api::PutCommand(makeDocumentBucket(bucket2), doc2, i + 10)), 0); - } - - { - FileStorHandler::LockedMessage lock = filestorHandler.getNextMessage(0, stripeId); - ASSERT_EQ(1, getPutTime(lock.second)); - - lock = filestorHandler.getNextMessage(0, stripeId, lock); - ASSERT_EQ(2, getPutTime(lock.second)); - - lock = filestorHandler.getNextMessage(0, stripeId, lock); - ASSERT_EQ(3, getPutTime(lock.second)); - } - - { - FileStorHandler::LockedMessage lock = filestorHandler.getNextMessage(0, stripeId); - ASSERT_EQ(11, getPutTime(lock.second)); - - lock = filestorHandler.getNextMessage(0, stripeId, lock); - ASSERT_EQ(12, getPutTime(lock.second)); - } -} - TEST_F(FileStorManagerTest, handler_timeout) { // Setup a filestorthread to test DummyStorageLink top; diff --git a/storage/src/tests/persistence/persistencequeuetest.cpp b/storage/src/tests/persistence/persistencequeuetest.cpp index be276dd7f9d..3754a82e7ae 100644 --- a/storage/src/tests/persistence/persistencequeuetest.cpp +++ b/storage/src/tests/persistence/persistencequeuetest.cpp @@ -168,19 +168,4 @@ TEST_F(PersistenceQueueTest, exclusive_locked_operation_not_started_if_exclusive ASSERT_FALSE(lock1.first.get()); } -TEST_F(PersistenceQueueTest, operation_batching_not_allowed_across_different_lock_modes) { - Fixture f(*this); - - f.filestorHandler->schedule(createPut(1234, 0), _disk); - f.filestorHandler->schedule(createGet(1234), _disk); - - auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId); - ASSERT_TRUE(lock0.first); - ASSERT_TRUE(lock0.second); - EXPECT_EQ(api::LockingRequirements::Exclusive, lock0.first->lockingRequirements()); - - f.filestorHandler->getNextMessage(_disk, f.stripeId, lock0); - ASSERT_FALSE(lock0.second); -} - } // namespace storage diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp index 543accc80ae..44cd8f0ab0c 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp @@ -60,12 +60,6 @@ FileStorHandler::getNextMessage(uint16_t disk, uint32_t stripeId) return _impl->getNextMessage(disk, stripeId); } -FileStorHandler::LockedMessage & -FileStorHandler::getNextMessage(uint16_t disk, uint32_t stripeId, LockedMessage& lck) -{ - return _impl->getNextMessage(disk, stripeId, lck); -} - FileStorHandler::BucketLockInterface::SP FileStorHandler::lock(const document::Bucket& bucket, uint16_t disk, api::LockingRequirements lockReq) { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index db294d7c39f..af0a52b2fa0 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -120,11 +120,6 @@ public: */ LockedMessage getNextMessage(uint16_t disk, uint32_t stripeId); - /** - * Returns the next message for the same bucket. - */ - LockedMessage & getNextMessage(uint16_t disk, uint32_t stripeId, LockedMessage& lock); - /** * Lock a bucket. By default, each file stor thread has the locks of all * buckets in their area of responsibility. If they need to access buckets diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index bfd5233d017..5ad37d585ef 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -319,25 +319,6 @@ FileStorHandlerImpl::getNextStripeId(uint32_t disk) { return _diskInfo[disk].getNextStripeId(); } - -FileStorHandler::LockedMessage & -FileStorHandlerImpl::getNextMessage(uint16_t diskId, uint32_t stripeId, FileStorHandler::LockedMessage& lck) -{ - document::Bucket bucket(lck.first->getBucket()); - - LOG(spam, "Disk %d retrieving message for buffered bucket %s", diskId, bucket.getBucketId().toString().c_str()); - - assert(diskId < _diskInfo.size()); - Disk& disk(_diskInfo[diskId]); - - if (disk.isClosed()) { - lck.second.reset(); - return lck; - } - - return disk.getNextMessage(stripeId, lck); -} - bool FileStorHandlerImpl::tryHandlePause(uint16_t disk) const { @@ -977,49 +958,6 @@ FileStorHandlerImpl::Stripe::getNextMessage(uint32_t timeout, Disk & disk) return {}; // No message fetched. } -FileStorHandler::LockedMessage & -FileStorHandlerImpl::Stripe::getNextMessage(FileStorHandler::LockedMessage& lck) -{ - const document::Bucket & bucket = lck.second->getBucket(); - vespalib::MonitorGuard guard(_lock); - BucketIdx& idx = bmi::get<2>(_queue); - std::pair range = idx.equal_range(bucket); - - // No more for this bucket. - if (range.first == range.second) { - lck.second.reset(); - return lck; - } - - api::StorageMessage & m(*range.first->_command); - // For now, don't allow batching of operations across lock requirement modes. - // We might relax this requirement later once we're 100% sure it can't trigger - // any unfortunate edge cases. - if (lck.first->lockingRequirements() != m.lockingRequirements()) { - lck.second.reset(); - return lck; - } - - std::chrono::milliseconds waitTime(uint64_t(range.first->_timer.stop(_metrics->averageQueueWaitingTime[m.getLoadType()]))); - - if (!messageTimedOutInQueue(m, waitTime)) { - std::shared_ptr msg = std::move(range.first->_command); - idx.erase(range.first); - lck.second.swap(msg); - guard.broadcast(); - } else { - std::shared_ptr msgReply = static_cast(m).makeReply(); - idx.erase(range.first); - guard.broadcast(); - guard.unlock(); - msgReply->setResult(api::ReturnCode(api::ReturnCode::TIMEOUT, "Message waited too long in storage queue")); - _messageSender.sendReply(msgReply); - - lck.second.reset(); - } - return lck; -} - FileStorHandler::LockedMessage FileStorHandlerImpl::Stripe::getMessage(vespalib::MonitorGuard & guard, PriorityIdx & idx, PriorityIdx::iterator iter) { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index da4d242a4c9..7a4f9000e82 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -129,7 +129,6 @@ public: void failOperations(const document::Bucket & bucket, const api::ReturnCode & code); FileStorHandler::LockedMessage getNextMessage(uint32_t timeout, Disk & disk); - FileStorHandler::LockedMessage & getNextMessage(FileStorHandler::LockedMessage& lock); void dumpQueue(std::ostream & os) const; void dumpActiveHtml(std::ostream & os) const; void dumpQueueHtml(std::ostream & os) const; @@ -185,9 +184,6 @@ public: FileStorHandler::LockedMessage getNextMessage(uint32_t stripeId, uint32_t timeout) { return _stripes[stripeId].getNextMessage(timeout, *this); } - FileStorHandler::LockedMessage & getNextMessage(uint32_t stripeId, FileStorHandler::LockedMessage & lck) { - return _stripes[stripeId].getNextMessage(lck); - } std::shared_ptr lock(const document::Bucket & bucket, api::LockingRequirements lockReq) { return stripe(bucket).lock(bucket, lockReq); @@ -253,8 +249,6 @@ public: FileStorHandler::LockedMessage getNextMessage(uint16_t disk, uint32_t stripeId); - FileStorHandler::LockedMessage & getNextMessage(uint16_t disk, uint32_t stripeId, FileStorHandler::LockedMessage& lock); - enum Operation { MOVE, SPLIT, JOIN }; void remapQueue(const RemapInfo& source, RemapInfo& target, Operation op); diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index e7123164659..77833c9af0d 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -894,115 +894,24 @@ bool hasBucketInfo(const api::StorageMessage& msg) } -void -PersistenceThread::flushAllReplies( - const document::Bucket& bucket, - std::vector >& replies) -{ - if (replies.empty()) { - return; - } - - try { - if (replies.size() > 1) { - _env._metrics.batchingSize.addValue(replies.size()); - } -#ifdef ENABLE_BUCKET_OPERATION_LOGGING - { - size_t nputs = 0, nremoves = 0, nother = 0; - for (size_t i = 0; i < replies.size(); ++i) { - if (dynamic_cast(replies[i]->getReply().get())) - { - ++nputs; - } else if (dynamic_cast( - replies[i]->getReply().get())) - { - ++nremoves; - } else { - ++nother; - } - } - LOG_BUCKET_OPERATION( - bucket.getBucketId(), - vespalib::make_string( - "flushing %zu operations (%zu puts, %zu removes, " - "%zu other)", - replies.size(), nputs, nremoves, nother)); - } -#endif - spi::Bucket b(bucket, spi::PartitionId(_env._partition)); - // Flush is not used for anything currentlu, and the context is not correct either when batching is done - // So just faking it here. - spi::Context dummyContext(documentapi::LoadType::DEFAULT, 0, 0); - spi::Result result = _spi.flush(b, dummyContext); - uint32_t errorCode = _env.convertErrorCode(result); - if (errorCode != 0) { - for (uint32_t i = 0; i < replies.size(); ++i) { - replies[i]->getReply()->setResult(api::ReturnCode((api::ReturnCode::Result)errorCode, result.getErrorMessage())); - } - } - } catch (std::exception& e) { - for (uint32_t i = 0; i < replies.size(); ++i) { - replies[i]->getReply()->setResult(api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE, e.what())); - } - } - - for (uint32_t i = 0; i < replies.size(); ++i) { - LOG(spam, "Sending reply up (batched): %s %" PRIu64, - replies[i]->getReply()->toString().c_str(), replies[i]->getReply()->getMsgId()); - _env._fileStorHandler.sendReply(replies[i]->getReply()); - } - - replies.clear(); -} - -void PersistenceThread::processMessages(FileStorHandler::LockedMessage & lock) -{ +void PersistenceThread::processMessages(FileStorHandler::LockedMessage & lock) { std::vector trackers; document::Bucket bucket = lock.first->getBucket(); - while (lock.second) { - LOG(debug, "Inside while loop %d, nodeIndex %d, ptr=%p", _env._partition, _env._nodeIndex, lock.second.get()); - std::shared_ptr msg(lock.second); - bool batchable = isBatchable(*msg); - - // If the next operation wasn't batchable, we should flush - // everything that came before. - if (!batchable) { - flushAllReplies(bucket, trackers); - } - - std::unique_ptr tracker = processMessage(*msg); - if (!tracker || !tracker->getReply()) { - // Was a reply - break; - } + LOG(debug, "Inside while loop %d, nodeIndex %d, ptr=%p", _env._partition, _env._nodeIndex, lock.second.get()); + std::shared_ptr msg(lock.second); + std::unique_ptr tracker = processMessage(*msg); + if (tracker && tracker->getReply()) { if (hasBucketInfo(*msg)) { if (tracker->getReply()->getResult().success()) { _env.setBucketInfo(*tracker, bucket); } } - if (batchable) { - LOG(spam, "Adding reply %s to batch for bucket %s", - tracker->getReply()->toString().c_str(), bucket.getBucketId().toString().c_str()); - - trackers.push_back(std::move(tracker)); - - if (trackers.back()->getReply()->getResult().success()) { - _env._fileStorHandler.getNextMessage(_env._partition, _stripeId, lock); - } else { - break; - } - } else { - LOG(spam, "Sending reply up: %s %" PRIu64, - tracker->getReply()->toString().c_str(), tracker->getReply()->getMsgId()); - _env._fileStorHandler.sendReply(tracker->getReply()); - break; - } + LOG(spam, "Sending reply up: %s %" PRIu64, + tracker->getReply()->toString().c_str(), tracker->getReply()->getMsgId()); + _env._fileStorHandler.sendReply(tracker->getReply()); } - - flushAllReplies(bucket, trackers); } void diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index e410843c1be..e331c66aeee 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -82,8 +82,6 @@ private: bool checkForError(const spi::Result& response, MessageTracker& tracker); spi::Bucket getBucket(const DocumentId& id, const document::Bucket &bucket) const; - void flushAllReplies(const document::Bucket& bucket, std::vector& trackers); - friend class TestAndSetHelper; bool tasConditionExists(const api::TestAndSetCommand & cmd); bool tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTracker & tracker, -- cgit v1.2.3 From 2e1602d453dc4272e7e1040a21f12e3438c8a5aa Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 27 Apr 2020 13:23:01 +0000 Subject: Update comments, and don't copy the shared ptr when not necessary. --- .../storage/persistence/filestorage/filestormanager.cpp | 2 +- storage/src/vespa/storage/persistence/persistencethread.cpp | 13 +++++++------ storage/src/vespa/storage/persistence/persistencethread.h | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index ea6f5e03a80..6eab44923f7 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -802,7 +802,7 @@ void FileStorManager::onFlush(bool downwards) LOG(debug, "Flushed _filestorHandler->flush(!downwards);"); for (uint32_t i = 0; i < _disks.size(); ++i) { for (uint32_t j = 0; j < _disks[i].size(); ++j) { - if (_disks[i][j].get() != NULL) { + if (_disks[i][j]) { _disks[i][j]->flush(); LOG(debug, "flushed disk[%d][%d]", i, j); } diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 77833c9af0d..d1da25269cf 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -894,16 +894,17 @@ bool hasBucketInfo(const api::StorageMessage& msg) } -void PersistenceThread::processMessages(FileStorHandler::LockedMessage & lock) { +void +PersistenceThread::processLockedMessage(FileStorHandler::LockedMessage & lock) { std::vector trackers; document::Bucket bucket = lock.first->getBucket(); - LOG(debug, "Inside while loop %d, nodeIndex %d, ptr=%p", _env._partition, _env._nodeIndex, lock.second.get()); - std::shared_ptr msg(lock.second); + LOG(debug, "Partition %d, nodeIndex %d, ptr=%p", _env._partition, _env._nodeIndex, lock.second.get()); + api::StorageMessage & msg(*lock.second); - std::unique_ptr tracker = processMessage(*msg); + std::unique_ptr tracker = processMessage(msg); if (tracker && tracker->getReply()) { - if (hasBucketInfo(*msg)) { + if (hasBucketInfo(msg)) { if (tracker->getReply()->getResult().success()) { _env.setBucketInfo(*tracker, bucket); } @@ -925,7 +926,7 @@ PersistenceThread::run(framework::ThreadHandle& thread) FileStorHandler::LockedMessage lock(_env._fileStorHandler.getNextMessage(_env._partition, _stripeId)); if (lock.first) { - processMessages(lock); + processLockedMessage(lock); } vespalib::MonitorGuard flushMonitorGuard(_flushMonitor); diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index e331c66aeee..56414835b7b 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -22,7 +22,7 @@ public: PersistenceThread(ServiceLayerComponentRegister&, const config::ConfigUri & configUri, spi::PersistenceProvider& provider, FileStorHandler& filestorHandler, FileStorThreadMetrics& metrics, uint16_t deviceIndex); - ~PersistenceThread(); + ~PersistenceThread() override; /** Waits for current operation to be finished. */ void flush() override; @@ -75,7 +75,7 @@ private: void handleReply(api::StorageReply&); MessageTracker::UP processMessage(api::StorageMessage& msg); - void processMessages(FileStorHandler::LockedMessage & lock); + void processLockedMessage(FileStorHandler::LockedMessage & lock); // Thread main loop void run(framework::ThreadHandle&) override; -- cgit v1.2.3 From f1a0b4686d58a4b76fd8cbe6fca710a904432e8b Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 27 Apr 2020 13:02:42 +0000 Subject: Remove flush from provider interface. --- .../conformancetest/conformancetest.cpp | 71 ---------------------- .../persistence/spi/abstractpersistenceprovider.h | 5 -- .../vespa/persistence/spi/persistenceprovider.h | 24 -------- searchcore/src/apps/proton/downpersistence.cpp | 6 -- searchcore/src/apps/proton/downpersistence.h | 1 - .../common/persistenceproviderwrapper.cpp | 9 --- .../common/persistenceproviderwrapper.h | 6 +- storage/src/tests/persistence/mergehandlertest.cpp | 13 +--- .../src/tests/persistence/persistencetestutils.cpp | 1 - .../src/vespa/storage/persistence/mergehandler.cpp | 38 ------------ .../storage/persistence/persistencethread.cpp | 4 -- .../storage/persistence/processallhandler.cpp | 17 ++---- .../storage/persistence/provider_error_wrapper.cpp | 6 -- .../storage/persistence/provider_error_wrapper.h | 1 - 14 files changed, 10 insertions(+), 192 deletions(-) diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 5c8b5b029d2..298e29cbecb 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -300,8 +300,6 @@ feedDocs(PersistenceProvider& spi, EXPECT_TRUE(!result.hasError()); docs.push_back(DocAndTimestamp(doc, Timestamp(1000 + i))); } - spi.flush(bucket, context); - EXPECT_EQ(Result(), Result(spi.flush(bucket, context))); return docs; } @@ -354,8 +352,6 @@ TEST_F(ConformanceTest, testBasics) Result(), Result(spi->remove(bucket, Timestamp(3), doc1->getId(), context))); - EXPECT_EQ(Result(), Result(spi->flush(bucket, context))); - // Iterate first without removes, then with. for (int iterPass = 0; iterPass < 2; ++iterPass) { bool includeRemoves = (iterPass == 1); @@ -441,11 +437,8 @@ TEST_F(ConformanceTest, testListBuckets) spi->createBucket(bucket3, context); spi->put(bucket1, Timestamp(1), doc1, context); - spi->flush(bucket1, context); spi->put(bucket2, Timestamp(2), doc2, context); - spi->flush(bucket2, context); spi->put(bucket3, Timestamp(3), doc3, context); - spi->flush(bucket3, context); { BucketIdListResult result = spi->listBuckets(makeBucketSpace(), PartitionId(1)); @@ -482,7 +475,6 @@ TEST_F(ConformanceTest, testBucketInfo) spi->put(bucket, Timestamp(2), doc2, context); const BucketInfo info1 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); { EXPECT_EQ(1, (int)info1.getDocumentCount()); @@ -492,7 +484,6 @@ TEST_F(ConformanceTest, testBucketInfo) spi->put(bucket, Timestamp(3), doc1, context); const BucketInfo info2 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); { EXPECT_EQ(2, (int)info2.getDocumentCount()); @@ -503,7 +494,6 @@ TEST_F(ConformanceTest, testBucketInfo) spi->put(bucket, Timestamp(4), doc1, context); const BucketInfo info3 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); { EXPECT_EQ(2, (int)info3.getDocumentCount()); @@ -514,7 +504,6 @@ TEST_F(ConformanceTest, testBucketInfo) spi->remove(bucket, Timestamp(5), doc1->getId(), context); const BucketInfo info4 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); { EXPECT_EQ(1, (int)info4.getDocumentCount()); @@ -544,7 +533,6 @@ TEST_F(ConformanceTest, testOrderIndependentBucketInfo) { spi->put(bucket, Timestamp(2), doc1, context); spi->put(bucket, Timestamp(3), doc2, context); - spi->flush(bucket, context); const BucketInfo info(spi->getBucketInfo(bucket).getBucketInfo()); checksumOrdered = info.getChecksum(); @@ -563,7 +551,6 @@ TEST_F(ConformanceTest, testOrderIndependentBucketInfo) // Swap order of puts spi->put(bucket, Timestamp(3), doc2, context); spi->put(bucket, Timestamp(2), doc1, context); - spi->flush(bucket, context); const BucketInfo info(spi->getBucketInfo(bucket).getBucketInfo()); checksumUnordered = info.getChecksum(); @@ -589,7 +576,6 @@ TEST_F(ConformanceTest, testPut) { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_TRUE(info.getEntryCount() >= info.getDocumentCount()); @@ -615,7 +601,6 @@ TEST_F(ConformanceTest, testPutNewDocumentVersion) Result result = spi->put(bucket, Timestamp(3), doc1, context); { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_TRUE(info.getEntryCount() >= info.getDocumentCount()); @@ -627,7 +612,6 @@ TEST_F(ConformanceTest, testPutNewDocumentVersion) result = spi->put(bucket, Timestamp(4), doc2, context); { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_TRUE(info.getEntryCount() >= info.getDocumentCount()); @@ -666,7 +650,6 @@ TEST_F(ConformanceTest, testPutOlderDocumentVersion) Result result = spi->put(bucket, Timestamp(5), doc1, context); const BucketInfo info1 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); { EXPECT_EQ(1, (int)info1.getDocumentCount()); EXPECT_TRUE(info1.getEntryCount() >= info1.getDocumentCount()); @@ -678,7 +661,6 @@ TEST_F(ConformanceTest, testPutOlderDocumentVersion) result = spi->put(bucket, Timestamp(4), doc2, context); { const BucketInfo info2 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info2.getDocumentCount()); EXPECT_TRUE(info2.getEntryCount() >= info1.getDocumentCount()); @@ -712,7 +694,6 @@ TEST_F(ConformanceTest, testPutDuplicate) BucketChecksum checksum; { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); checksum = info.getChecksum(); } @@ -721,7 +702,6 @@ TEST_F(ConformanceTest, testPutDuplicate) { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_EQ(checksum, info.getChecksum()); } @@ -746,7 +726,6 @@ TEST_F(ConformanceTest, testRemove) { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_TRUE(info.getChecksum() != 0); @@ -764,7 +743,6 @@ TEST_F(ConformanceTest, testRemove) { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(0, (int)info.getDocumentCount()); EXPECT_EQ(0, (int)info.getChecksum()); @@ -791,7 +769,6 @@ TEST_F(ConformanceTest, testRemove) context); { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(0, (int)info.getDocumentCount()); EXPECT_EQ(0, (int)info.getChecksum()); @@ -799,7 +776,6 @@ TEST_F(ConformanceTest, testRemove) } Result result4 = spi->put(bucket, Timestamp(9), doc1, context); - spi->flush(bucket, context); EXPECT_TRUE(!result4.hasError()); @@ -809,7 +785,6 @@ TEST_F(ConformanceTest, testRemove) context); { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(0, (int)info.getDocumentCount()); EXPECT_EQ(0, (int)info.getChecksum()); @@ -847,7 +822,6 @@ TEST_F(ConformanceTest, testRemoveMerge) Timestamp(10), removeId, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, removeResult.getErrorCode()); EXPECT_EQ(false, removeResult.wasFound()); } @@ -875,7 +849,6 @@ TEST_F(ConformanceTest, testRemoveMerge) Timestamp(11), removeId, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, removeResult.getErrorCode()); EXPECT_EQ(false, removeResult.wasFound()); } @@ -903,7 +876,6 @@ TEST_F(ConformanceTest, testRemoveMerge) Timestamp(7), removeId, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, removeResult.getErrorCode()); EXPECT_EQ(false, removeResult.wasFound()); } @@ -946,7 +918,6 @@ TEST_F(ConformanceTest, testUpdate) { UpdateResult result = spi->update(bucket, Timestamp(3), update, context); - spi->flush(bucket, context); EXPECT_EQ(Result(), Result(result)); EXPECT_EQ(Timestamp(0), result.getExistingTimestamp()); } @@ -955,7 +926,6 @@ TEST_F(ConformanceTest, testUpdate) { UpdateResult result = spi->update(bucket, Timestamp(4), update, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(Timestamp(3), result.getExistingTimestamp()); @@ -975,7 +945,6 @@ TEST_F(ConformanceTest, testUpdate) } spi->remove(bucket, Timestamp(5), doc1->getId(), context); - spi->flush(bucket, context); { GetResult result = spi->get(bucket, @@ -991,7 +960,6 @@ TEST_F(ConformanceTest, testUpdate) { UpdateResult result = spi->update(bucket, Timestamp(6), update, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(Timestamp(0), result.getExistingTimestamp()); @@ -1009,7 +977,6 @@ TEST_F(ConformanceTest, testUpdate) // Document does not exist (and therefore its condition cannot match by definition), // but since CreateIfNonExistent is set it should be auto-created anyway. UpdateResult result = spi->update(bucket, Timestamp(7), update, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(Timestamp(7), result.getExistingTimestamp()); } @@ -1044,7 +1011,6 @@ TEST_F(ConformanceTest, testGet) } spi->put(bucket, Timestamp(3), doc1, context); - spi->flush(bucket, context); { GetResult result = spi->get(bucket, document::AllFields(), @@ -1054,7 +1020,6 @@ TEST_F(ConformanceTest, testGet) } spi->remove(bucket, Timestamp(4), doc1->getId(), context); - spi->flush(bucket, context); { GetResult result = spi->get(bucket, document::AllFields(), @@ -1168,7 +1133,6 @@ TEST_F(ConformanceTest, testIterateAllDocsNewestVersionOnly) spi->put(b, newTimestamp, newDoc, context); newDocs.push_back(DocAndTimestamp(newDoc, newTimestamp)); } - spi->flush(b, context); CreateIteratorResult iter(createIterator(*spi, b, createSelection(""))); @@ -1249,7 +1213,6 @@ TEST_F(ConformanceTest, testIterateMatchTimestampRange) DocAndTimestamp(doc, Timestamp(1000 + i))); } } - spi->flush(b, context); Selection sel = Selection(DocumentSelection("")); sel.setFromTimestamp(fromTimestamp); @@ -1295,7 +1258,6 @@ TEST_F(ConformanceTest, testIterateExplicitTimestampSubset) Timestamp(2000), docsToVisit.front().doc->getId(), context) .wasFound()); - spi->flush(b, context); timestampsToVisit.push_back(Timestamp(2000)); removes.insert(docsToVisit.front().doc->getId().toString()); @@ -1339,7 +1301,6 @@ TEST_F(ConformanceTest, testIterateRemoves) nonRemovedDocs.push_back(docs[i]); } } - spi->flush(b, context); // First, test iteration without removes { @@ -1387,7 +1348,6 @@ TEST_F(ConformanceTest, testIterateMatchSelection) DocAndTimestamp(doc, Timestamp(1000 + i))); } } - spi->flush(b, context); CreateIteratorResult iter( createIterator(*spi, @@ -1416,7 +1376,6 @@ TEST_F(ConformanceTest, testIterationRequiringDocumentIdOnlyMatching) // remove entry for it regardless. EXPECT_TRUE( !spi->remove(b, Timestamp(2000), removedId, context).wasFound()); - spi->flush(b, context); Selection sel(createSelection("id == '" + removedId.toString() + "'")); @@ -1530,7 +1489,6 @@ TEST_F(ConformanceTest, testDeleteBucket) spi->createBucket(bucket, context); spi->put(bucket, Timestamp(3), doc1, context); - spi->flush(bucket, context); spi->deleteBucket(bucket, context); testDeleteBucketPostCondition(spi, bucket, *doc1); @@ -1586,8 +1544,6 @@ TEST_F(ConformanceTest, testSplitNormalCase) spi->put(bucketC, Timestamp(i + 1), doc1, context); } - spi->flush(bucketC, context); - spi->split(bucketC, bucketA, bucketB, context); testSplitNormalCasePostCondition(spi, bucketA, bucketB, bucketC, testDocMan); @@ -1657,28 +1613,23 @@ TEST_F(ConformanceTest, testSplitTargetExists) spi->put(bucketC, Timestamp(i + 1), doc1, context); } - spi->flush(bucketC, context); for (uint32_t i = 10; i < 20; ++i) { Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); spi->put(bucketB, Timestamp(i + 1), doc1, context); } - spi->flush(bucketB, context); EXPECT_TRUE(!spi->getBucketInfo(bucketB).getBucketInfo().isActive()); for (uint32_t i = 10; i < 20; ++i) { Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); spi->put(bucketC, Timestamp(i + 1), doc1, context); } - spi->flush(bucketC, context); for (uint32_t i = 20; i < 25; ++i) { Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); spi->put(bucketB, Timestamp(i + 1), doc1, context); } - spi->flush(bucketB, context); - spi->split(bucketC, bucketA, bucketB, context); testSplitTargetExistsPostCondition(spi, bucketA, bucketB, bucketC, testDocMan); @@ -1745,8 +1696,6 @@ TEST_F(ConformanceTest, testSplitSingleDocumentInSource) Document::SP doc = testDocMan.createRandomDocumentAtLocation(0x06, 0); spi->put(source, Timestamp(1), doc, context); - spi->flush(source, context); - spi->split(source, target1, target2, context); testSplitSingleDocumentInSourcePostCondition( spi, source, target1, target2, testDocMan); @@ -1804,7 +1753,6 @@ ConformanceTest::createAndPopulateJoinSourceBuckets( source1.getBucketId().getId(), i)); spi.put(source1, Timestamp(i + 1), doc, context); } - spi.flush(source1, context); for (uint32_t i = 10; i < 20; ++i) { Document::SP doc( @@ -1812,7 +1760,6 @@ ConformanceTest::createAndPopulateJoinSourceBuckets( source2.getBucketId().getId(), i)); spi.put(source2, Timestamp(i + 1), doc, context); } - spi.flush(source2, context); } void @@ -1912,21 +1859,17 @@ TEST_F(ConformanceTest, testJoinTargetExists) spi->put(bucketA, Timestamp(i + 1), doc1, context); } - spi->flush(bucketA, context); for (uint32_t i = 10; i < 20; ++i) { Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); spi->put(bucketB, Timestamp(i + 1), doc1, context); } - spi->flush(bucketB, context); for (uint32_t i = 20; i < 30; ++i) { Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); spi->put(bucketC, Timestamp(i + 1), doc1, context); } - spi->flush(bucketC, context); - spi->join(bucketA, bucketB, bucketC, context); testJoinTargetExistsPostCondition(spi, bucketA, bucketB, bucketC, testDocMan); @@ -1991,7 +1934,6 @@ ConformanceTest::populateBucket(const Bucket& b, location, i); spi.put(b, Timestamp(i + 1), doc1, context); } - spi.flush(b, context); } TEST_F(ConformanceTest, testJoinOneBucket) @@ -2150,7 +2092,6 @@ TEST_F(ConformanceTest, testMaintain) spi->createBucket(bucket, context); spi->put(bucket, Timestamp(3), doc1, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, spi->maintain(bucket, LOW).getErrorCode()); @@ -2229,7 +2170,6 @@ TEST_F(SingleDocTypeConformanceTest, testBucketActivationSplitAndJoin) spi->createBucket(bucketC, context); spi->put(bucketC, Timestamp(1), doc1, context); spi->put(bucketC, Timestamp(2), doc2, context); - spi->flush(bucketC, context); spi->setActiveState(bucketC, BucketInfo::ACTIVE); EXPECT_TRUE(spi->getBucketInfo(bucketC).getBucketInfo().isActive()); @@ -2304,14 +2244,11 @@ TEST_F(ConformanceTest, testRemoveEntry) spi->createBucket(bucket, context); spi->put(bucket, Timestamp(3), doc1, context); - spi->flush(bucket, context); BucketInfo info1 = spi->getBucketInfo(bucket).getBucketInfo(); { spi->put(bucket, Timestamp(4), doc2, context); - spi->flush(bucket, context); spi->removeEntry(bucket, Timestamp(4), context); - spi->flush(bucket, context); BucketInfo info2 = spi->getBucketInfo(bucket).getBucketInfo(); EXPECT_EQ(info1, info2); } @@ -2319,9 +2256,7 @@ TEST_F(ConformanceTest, testRemoveEntry) // Test case where there exists a previous version of the document. { spi->put(bucket, Timestamp(5), doc1, context); - spi->flush(bucket, context); spi->removeEntry(bucket, Timestamp(5), context); - spi->flush(bucket, context); BucketInfo info2 = spi->getBucketInfo(bucket).getBucketInfo(); EXPECT_EQ(info1, info2); } @@ -2329,14 +2264,11 @@ TEST_F(ConformanceTest, testRemoveEntry) // Test case where the newest document version after removeEntrying is a remove. { spi->remove(bucket, Timestamp(6), doc1->getId(), context); - spi->flush(bucket, context); BucketInfo info2 = spi->getBucketInfo(bucket).getBucketInfo(); EXPECT_EQ(uint32_t(0), info2.getDocumentCount()); spi->put(bucket, Timestamp(7), doc1, context); - spi->flush(bucket, context); spi->removeEntry(bucket, Timestamp(7), context); - spi->flush(bucket, context); BucketInfo info3 = spi->getBucketInfo(bucket).getBucketInfo(); EXPECT_EQ(info2, info3); } @@ -2395,9 +2327,6 @@ TEST_F(ConformanceTest, testBucketSpaces) spi->put(bucket01, Timestamp(4), doc2, context); spi->put(bucket11, Timestamp(5), doc3, context); spi->put(bucket12, Timestamp(6), doc4, context); - spi->flush(bucket01, context); - spi->flush(bucket11, context); - spi->flush(bucket12, context); // Check bucket lists assertBucketList(*spi, bucketSpace0, partId, { bucketId1 }); assertBucketList(*spi, bucketSpace1, partId, { bucketId1, bucketId2 }); diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h index 557a9ec2edd..cbd76951004 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h @@ -40,11 +40,6 @@ public: */ Result removeEntry(const Bucket&, Timestamp, Context&) override { return Result(); } - /** - * Default impl is getBucketInfo(); - */ - Result flush(const Bucket&, Context&) override { return Result(); } - /** * Default impl is remove(). */ diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index 857630e2606..c70d5e3f1c3 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -201,30 +201,6 @@ struct PersistenceProvider */ virtual UpdateResult update(const Bucket&, Timestamp timestamp, const DocumentUpdateSP& update, Context&) = 0; - /** - * The service layer may choose to batch certain commands. This means that - * the service layer will lock the bucket only once, then perform several - * commands, and finally get the bucket info from the bucket, and then - * flush it. This can be used to improve performance by caching the - * modifications, and persisting them to disk only when flush is called. - * The service layer guarantees that after one of these operations, flush() - * is called, regardless of whether the operation succeeded or not, before - * another bucket is processed in the same worker thead. The following - * operations can be batched and have the guarantees - * above: - * - put - * - get - * - remove (all versions) - * - update - * - revert - * - join - *

- * A provider may of course choose to not sync to disk at flush time either, - * but then data may be more prone to being lost on node issues, and the - * provider must figure out when to flush its cache itself. - */ - virtual Result flush(const Bucket&, Context&) = 0; - /** * Retrieves the latest version of the document specified by the * document id. If no versions were found, or the document was removed, diff --git a/searchcore/src/apps/proton/downpersistence.cpp b/searchcore/src/apps/proton/downpersistence.cpp index 4b911eb6d2b..d4ec9cee395 100644 --- a/searchcore/src/apps/proton/downpersistence.cpp +++ b/searchcore/src/apps/proton/downpersistence.cpp @@ -98,12 +98,6 @@ UpdateResult DownPersistence::update(const Bucket&, Timestamp, errorResult.getErrorMessage()); } -Result -DownPersistence::flush(const Bucket&, Context&) -{ - return errorResult; -} - GetResult DownPersistence::get(const Bucket&, const document::FieldSet&, const DocumentId&, Context&) const diff --git a/searchcore/src/apps/proton/downpersistence.h b/searchcore/src/apps/proton/downpersistence.h index 0a602c4467e..8cdac7aaa1b 100644 --- a/searchcore/src/apps/proton/downpersistence.h +++ b/searchcore/src/apps/proton/downpersistence.h @@ -37,7 +37,6 @@ public: RemoveResult removeIfFound(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&) override; Result removeEntry(const Bucket&, Timestamp, Context&) override; UpdateResult update(const Bucket&, Timestamp timestamp, const DocumentUpdateSP& update, Context&) override; - Result flush(const Bucket&, Context&) override; GetResult get(const Bucket&, const document::FieldSet& fieldSet, const DocumentId& id, Context&) const override; CreateIteratorResult createIterator(const Bucket&, const document::FieldSet& fieldSet, diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp index 05e51993f49..fb80c25bfb7 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp @@ -145,15 +145,6 @@ PersistenceProviderWrapper::get(const spi::Bucket& bucket, return _spi.get(bucket, fieldSet, id, context); } -spi::Result -PersistenceProviderWrapper::flush(const spi::Bucket& bucket, - spi::Context& context) -{ - LOG_SPI("flush(" << bucket << ")"); - CHECK_ERROR(spi::Result, FAIL_FLUSH); - return _spi.flush(bucket, context); -} - spi::CreateIteratorResult PersistenceProviderWrapper::createIterator(const spi::Bucket& bucket, const document::FieldSet& fields, diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.h b/storage/src/tests/persistence/common/persistenceproviderwrapper.h index 511ced02118..9bd3653e8a1 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.h +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.h @@ -36,7 +36,6 @@ public: FAIL_REPLACE_WITH_REMOVE = 1 << 6, FAIL_UPDATE = 1 << 7, FAIL_REVERT = 1 << 8, - FAIL_FLUSH = 1 << 9, FAIL_CREATE_ITERATOR = 1 << 10, FAIL_ITERATE = 1 << 11, FAIL_DESTROY_ITERATOR = 1 << 12, @@ -44,7 +43,7 @@ public: FAIL_SPLIT = 1 << 14, FAIL_JOIN = 1 << 15, FAIL_CREATE_BUCKET = 1 << 16, - FAIL_BUCKET_PERSISTENCE = FAIL_PUT|FAIL_REMOVE|FAIL_UPDATE|FAIL_REVERT|FAIL_FLUSH, + FAIL_BUCKET_PERSISTENCE = FAIL_PUT|FAIL_REMOVE|FAIL_UPDATE|FAIL_REVERT, FAIL_ALL_OPERATIONS = 0xffff, // TODO: add more as needed }; @@ -55,7 +54,7 @@ private: uint32_t _failureMask; public: PersistenceProviderWrapper(spi::PersistenceProvider& spi); - ~PersistenceProviderWrapper(); + ~PersistenceProviderWrapper() override; /** * Explicitly set result to anything != NONE to have all operations @@ -96,7 +95,6 @@ public: spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const spi::DocumentId&, spi::Context&) const override ; - spi::Result flush(const spi::Bucket&, spi::Context&) override; spi::CreateIteratorResult createIterator(const spi::Bucket&, const document::FieldSet&, const spi::Selection&, spi::IncludedVersions versions, spi::Context&) override; diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index 0b2baab5652..7e7a76b95a8 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -626,8 +626,7 @@ MergeHandlerTest::createDummyGetBucketDiff(int timestampOffset, } TEST_F(MergeHandlerTest, spi_flush_guard) { - PersistenceProviderWrapper providerWrapper( - getPersistenceProvider()); + PersistenceProviderWrapper providerWrapper(getPersistenceProvider()); MergeHandler handler(providerWrapper, getEnv()); providerWrapper.setResult( @@ -635,8 +634,7 @@ TEST_F(MergeHandlerTest, spi_flush_guard) { setUpChain(MIDDLE); // Fail applying unrevertable remove - providerWrapper.setFailureMask( - PersistenceProviderWrapper::FAIL_REMOVE); + providerWrapper.setFailureMask(PersistenceProviderWrapper::FAIL_REMOVE); providerWrapper.clearOperationLog(); try { @@ -645,11 +643,6 @@ TEST_F(MergeHandlerTest, spi_flush_guard) { } catch (const std::runtime_error& e) { EXPECT_TRUE(std::string(e.what()).find("Failed remove") != std::string::npos); } - // Test that we always flush after applying diff locally, even when - // errors are encountered. - const std::vector& opLog(providerWrapper.getOperationLog()); - ASSERT_FALSE(opLog.empty()); - EXPECT_EQ("flush(Bucket(0x40000000000004d2, partition 0))", opLog.back()); } TEST_F(MergeHandlerTest, bucket_not_found_in_db) { @@ -901,7 +894,6 @@ TEST_F(MergeHandlerTest, apply_bucket_diff_spi_failures) { { PersistenceProviderWrapper::FAIL_ITERATE, "iterate" }, { PersistenceProviderWrapper::FAIL_PUT, "Failed put" }, { PersistenceProviderWrapper::FAIL_REMOVE, "Failed remove" }, - { PersistenceProviderWrapper::FAIL_FLUSH, "Failed flush" }, }; typedef ExpectedExceptionSpec* ExceptionIterator; @@ -1058,7 +1050,6 @@ TEST_F(MergeHandlerTest, apply_bucket_diff_reply_spi_failures) { { PersistenceProviderWrapper::FAIL_ITERATE, "iterate" }, { PersistenceProviderWrapper::FAIL_PUT, "Failed put" }, { PersistenceProviderWrapper::FAIL_REMOVE, "Failed remove" }, - { PersistenceProviderWrapper::FAIL_FLUSH, "Failed flush" }, }; typedef ExpectedExceptionSpec* ExceptionIterator; diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index e32fc056413..25c0a36a7f5 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -159,7 +159,6 @@ PersistenceTestUtils::doPutOnDisk( getPersistenceProvider().put(spi::Bucket(b), timestamp, doc, context); - getPersistenceProvider().flush(b, context); return doc; } diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index e5e358cbb60..d47ed28f636 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -91,40 +91,6 @@ public: } }; -class FlushGuard -{ - spi::PersistenceProvider& _spi; - spi::Bucket _bucket; - spi::Context& _context; - bool _hasFlushed; -public: - FlushGuard(spi::PersistenceProvider& spi, - const spi::Bucket& bucket, - spi::Context& context) - : _spi(spi), - _bucket(bucket), - _context(context), - _hasFlushed(false) - {} - ~FlushGuard() - { - if (!_hasFlushed) { - LOG(debug, "Auto-flushing %s", _bucket.toString().c_str()); - spi::Result result =_spi.flush(_bucket, _context); - if (result.hasError()) { - LOG(debug, "Flush %s failed: %s", - _bucket.toString().c_str(), - result.toString().c_str()); - } - } - } - void flush() { - LOG(debug, "Flushing %s", _bucket.toString().c_str()); - _hasFlushed = true; - checkResult(_spi.flush(_bucket, _context), _bucket, "flush"); - } -}; - struct IndirectDocEntryTimestampPredicate { bool operator()(const spi::DocEntry::UP& e1, @@ -607,8 +573,6 @@ MergeHandler::applyDiffLocally( std::vector entries; populateMetaData(bucket, MAX_TIMESTAMP, entries, context); - FlushGuard flushGuard(_spi, bucket, context); - std::shared_ptr repo(_env._component.getTypeRepo()); assert(repo.get() != nullptr); @@ -702,8 +666,6 @@ MergeHandler::applyDiffLocally( LOG(debug, "Merge(%s): Applied %u entries locally from ApplyBucketDiff.", bucket.toString().c_str(), addedCount); - flushGuard.flush(); - spi::BucketInfoResult infoResult(_spi.getBucketInfo(bucket)); if (infoResult.getErrorCode() != spi::Result::ErrorType::NONE) { LOG(warning, "Failed to get bucket info for %s: %s", diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index d1da25269cf..aaf62a85e87 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -607,10 +607,6 @@ PersistenceThread::handleJoinBuckets(api::JoinBucketsCommand& cmd, spi::Context if (!checkForError(result, *tracker)) { return tracker; } - result = _spi.flush(spi::Bucket(destBucket, spi::PartitionId(_env._partition)), context); - if (!checkForError(result, *tracker)) { - return tracker; - } uint64_t lastModified = 0; for (uint32_t i = 0; i < cmd.getSourceBuckets().size(); i++) { document::Bucket srcBucket(destBucket.getBucketSpace(), cmd.getSourceBuckets()[i]); diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp index 5b94a3da027..f37c6723933 100644 --- a/storage/src/vespa/storage/persistence/processallhandler.cpp +++ b/storage/src/vespa/storage/persistence/processallhandler.cpp @@ -83,9 +83,9 @@ MessageTracker::UP ProcessAllHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd, spi::Context& context) { - MessageTracker::UP tracker(new MessageTracker( + auto tracker = std::make_unique( _env._metrics.removeLocation[cmd.getLoadType()], - _env._component.getClock())); + _env._component.getClock()); LOG(debug, "RemoveLocation(%s): using selection '%s'", cmd.getBucketId().toString().c_str(), @@ -99,13 +99,8 @@ ProcessAllHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd, processor, spi::NEWEST_DOCUMENT_ONLY, context); - spi::Result result = _spi.flush(bucket, context); - uint32_t code = _env.convertErrorCode(result); - if (code == 0) { - tracker->setReply(std::make_shared(cmd, processor._n_removed)); - } else { - tracker->fail(code, result.getErrorMessage()); - } + + tracker->setReply(std::make_shared(cmd, processor._n_removed)); return tracker; } @@ -114,9 +109,9 @@ MessageTracker::UP ProcessAllHandler::handleStatBucket(api::StatBucketCommand& cmd, spi::Context& context) { - MessageTracker::UP tracker(new MessageTracker( + auto tracker = std::make_unique( _env._metrics.statBucket[cmd.getLoadType()], - _env._component.getClock())); + _env._component.getClock()); std::ostringstream ost; ost << "Persistence bucket " << cmd.getBucketId() diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index fe947bff4d5..76c420e76e6 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -118,12 +118,6 @@ ProviderErrorWrapper::get(const spi::Bucket& bucket, return checkResult(_impl.get(bucket, fieldSet, docId, context)); } -spi::Result -ProviderErrorWrapper::flush(const spi::Bucket& bucket, spi::Context& context) -{ - return checkResult(_impl.flush(bucket, context)); -} - spi::CreateIteratorResult ProviderErrorWrapper::createIterator(const spi::Bucket& bucket, const document::FieldSet& fieldSet, diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h index 3b5ace90d13..292eb004223 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h @@ -52,7 +52,6 @@ public: spi::RemoveResult removeIfFound(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::Context&) override; spi::UpdateResult update(const spi::Bucket&, spi::Timestamp, const spi::DocumentUpdateSP&, spi::Context&) override; spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const document::DocumentId&, spi::Context&) const override; - spi::Result flush(const spi::Bucket&, spi::Context&) override; spi::CreateIteratorResult createIterator(const spi::Bucket&, const document::FieldSet&, const spi::Selection&, spi::IncludedVersions versions, spi::Context&) override; spi::IterateResult iterate(spi::IteratorId, uint64_t maxByteSize, spi::Context&) const override; -- cgit v1.2.3 From 8e285a592262432769522ad769ce036056517b11 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Mon, 27 Apr 2020 15:54:21 +0200 Subject: Add container name to metadata --- .../com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index ef515022ae6..f203137383e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -157,6 +157,7 @@ public class StorageMaintainer { attributes.put("flavor", context.node().flavor()); attributes.put("kernel_version", System.getProperty("os.version")); attributes.put("cpu_microcode_version", getMicrocodeVersion()); + attributes.put("container_name", context.containerName().asString()); container.map(c -> c.image).ifPresent(image -> attributes.put("docker_image", image.asString())); context.node().parentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); -- cgit v1.2.3 From 0fa1ef44db7b7a98a2b026e5aada0b5daa7c16b4 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Mon, 27 Apr 2020 16:59:53 +0200 Subject: Use coredump path instead of container name --- .../yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java | 4 +++- .../vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index f203137383e..a1ad22294ce 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -157,7 +157,9 @@ public class StorageMaintainer { attributes.put("flavor", context.node().flavor()); attributes.put("kernel_version", System.getProperty("os.version")); attributes.put("cpu_microcode_version", getMicrocodeVersion()); - attributes.put("container_name", context.containerName().asString()); + attributes.put("coredump_path", coredumpHandler.getDoneCoredumpsPath() + .resolve(context.containerName().asString()) + .toString()); container.map(c -> c.image).ifPresent(image -> attributes.put("docker_image", image.asString())); context.node().parentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 9d20d973676..8bbde4bc298 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -255,4 +255,7 @@ public class CoredumpHandler { return dimensionsBuilder.build(); } + public Path getDoneCoredumpsPath() { + return this.doneCoredumpsPath; + } } -- cgit v1.2.3 From 25a4cb12c650b2f2154a864aacd5daac808e2b52 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Mon, 27 Apr 2020 17:27:56 +0200 Subject: Use full coredump path --- .../yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java | 3 --- .../vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java | 1 + .../hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index a1ad22294ce..ef515022ae6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -157,9 +157,6 @@ public class StorageMaintainer { attributes.put("flavor", context.node().flavor()); attributes.put("kernel_version", System.getProperty("os.version")); attributes.put("cpu_microcode_version", getMicrocodeVersion()); - attributes.put("coredump_path", coredumpHandler.getDoneCoredumpsPath() - .resolve(context.containerName().asString()) - .toString()); container.map(c -> c.image).ifPresent(image -> attributes.put("docker_image", image.asString())); context.node().parentHostname().ifPresent(parent -> attributes.put("parent_hostname", parent)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 8bbde4bc298..d24989f036f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -164,6 +164,7 @@ public class CoredumpHandler { Path coredumpFilePathInContainer = context.pathInNodeFromPathOnHost(coredumpFilePathOnHost); Map metadata = coreCollector.collect(context, coredumpFilePathInContainer); metadata.putAll(nodeAttributesSupplier.get()); + metadata.put("coredump_path", doneCoredumpsPath.resolve(context.containerName().asString()).resolve(coredumpDirectory.getFileName()).toString()); String metadataFields = objectMapper.writeValueAsString(Map.of("fields", metadata)); metadataPath.writeUtf8File(metadataFields); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index 60a4462e9e9..8aa4fb9131b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -159,6 +159,7 @@ public class CoredumpHandlerTest { "\"backtrace\":[\"call 1\",\"function 2\",\"something something\"]," + "\"vespa_version\":\"6.48.4\"," + "\"bin_path\":\"/bin/bash\"," + + "\"coredump_path\":\"/home/docker/dumps/container-123/id-123\"," + "\"docker_image\":\"vespa/ci:6.48.4\"" + "}}"; -- cgit v1.2.3 From bc47d04f90a696fa3c806c33f7980602ece8b9b4 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Mon, 27 Apr 2020 17:33:34 +0200 Subject: Remove unused function --- .../vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index d24989f036f..c6c024ca658 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -256,7 +256,4 @@ public class CoredumpHandler { return dimensionsBuilder.build(); } - public Path getDoneCoredumpsPath() { - return this.doneCoredumpsPath; - } } -- cgit v1.2.3