diff options
244 files changed, 819 insertions, 928 deletions
diff --git a/Code-of-Conduct.md b/Code-of-Conduct.md index 00e20d0f018..59729463e62 100644 --- a/Code-of-Conduct.md +++ b/Code-of-Conduct.md @@ -1,4 +1,4 @@ -<!-- Copyright 2020 Verizon Media Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # Verizon Media Open Source Code of Conduct diff --git a/ERRATA.md b/ERRATA.md index 9278f3c39cf..5091fb70d6a 100644 --- a/ERRATA.md +++ b/ERRATA.md @@ -1,4 +1,4 @@ -<!-- Copyright verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> ## Errata diff --git a/client/README.md b/client/README.md index 39d9bd6b669..047276455ec 100644 --- a/client/README.md +++ b/client/README.md @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # vespa_query_dsl This lib is used for composing vespa YQL queries diff --git a/client/pom.xml b/client/pom.xml index 6f281fddf8c..ccf980f11dc 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -1,4 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> diff --git a/cloud-tenant-base-dependencies-enforcer/README.md b/cloud-tenant-base-dependencies-enforcer/README.md index 46d1af7090e..9708c6b9441 100644 --- a/cloud-tenant-base-dependencies-enforcer/README.md +++ b/cloud-tenant-base-dependencies-enforcer/README.md @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # Dependencies enforcer for cloud-tenant-base parent pom Enforces that only allowed dependencies are visible for tenant projects using the cloud-tenant-base parent pom. diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 898ebe243b0..5d5ffb917d2 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -597,10 +597,6 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd public void tick() throws Exception { synchronized (monitor) { boolean didWork; - // We defer ZooKeeper connections until our local Slobrok mirror is ready. - // Otherwise we risk winning an election without having any clue about the state of - // the nodes in the cluster, causing us to spuriously publish cluster down-states. - database.setConnectionEstablishmentIsAllowed(nodeLookup.isReady()); didWork = database.doNextZooKeeperTask(databaseContext); didWork |= updateMasterElectionState(); didWork |= handleLeadershipEdgeTransitions(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java index 65b97a3ae82..ceb81e91b7d 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java @@ -12,11 +12,4 @@ public interface NodeLookup { boolean updateCluster(ContentCluster cluster, NodeAddedOrRemovedListener listener); - /** - * Returns whether the lookup instance has been able to bootstrap itself with information about nodes. - * - * Calling updateCluster() _before_ isReady has returned true may not provide any useful data. - */ - boolean isReady(); - } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java index 8d0a873a801..531e29344a6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java @@ -5,6 +5,7 @@ import com.yahoo.jrt.Spec; import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.core.hostinfo.ResourceUsage; +import java.util.Locale; import java.util.Objects; /** @@ -46,22 +47,22 @@ public class NodeResourceExhaustion { } public String toExhaustionAddedDescription() { - return String.format("%s (%.3g > %.3g)", makeDescriptionPrefix(), resourceUsage.getUsage(), limit); + return String.format(Locale.US, "%s (%.3g > %.3g)", makeDescriptionPrefix(), resourceUsage.getUsage(), limit); } public String toExhaustionRemovedDescription() { - return String.format("%s (<= %.3g)", makeDescriptionPrefix(), limit); + return String.format(Locale.US, "%s (<= %.3g)", makeDescriptionPrefix(), limit); } public String toShorthandDescription() { - return String.format("%s%s %.3g > %.3g", + return String.format(Locale.US, "%s%s %.3g > %.3g", resourceType, (resourceUsage.getName() != null ? ":" + resourceUsage.getName() : ""), resourceUsage.getUsage(), limit); } private String makeDescriptionPrefix() { - return String.format("%s%s on node %d [%s]", + return String.format(Locale.US, "%s%s on node %d [%s]", resourceType, (resourceUsage.getName() != null ? ":" + resourceUsage.getName() : ""), node.getIndex(), diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java index 654a2aaae7f..4f02e31b426 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java @@ -91,7 +91,6 @@ public class DatabaseHandler { private long lastZooKeeperConnectionAttempt = 0; private static final int minimumWaitBetweenFailedConnectionAttempts = 10000; private boolean lostZooKeeperConnectionEvent = false; - private boolean connectionEstablishmentIsAllowed = false; private Map<Integer, Integer> masterDataEvent = null; public DatabaseHandler(DatabaseFactory databaseFactory, Timer timer, String zooKeeperAddress, int ourIndex, Object monitor) throws InterruptedException @@ -243,7 +242,7 @@ public class DatabaseHandler { didWork = true; } } - if (isDatabaseClosedSafe() && zooKeeperIsConfigured() && connectionEstablishmentIsAllowed) { + if (isDatabaseClosedSafe() && zooKeeperIsConfigured()) { long currentTime = timer.getCurrentTimeInMillis(); if (currentTime - lastZooKeeperConnectionAttempt < minimumWaitBetweenFailedConnectionAttempts) { return false; // Not time to attempt connection yet. @@ -268,10 +267,6 @@ public class DatabaseHandler { return didWork; } - public void setConnectionEstablishmentIsAllowed(boolean allowed) { - connectionEstablishmentIsAllowed = allowed; - } - private boolean zooKeeperIsConfigured() { // This should only ever be null during unit testing. return zooKeeperAddress != null; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index 8649e7cc11a..b3bb458ed74 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -59,7 +59,6 @@ public class SlobrokClient implements NodeLookup { freshMirror = true; } - @Override public void shutdown() { if (supervisor != null) { supervisor.transport().shutdown().join(); @@ -68,12 +67,6 @@ public class SlobrokClient implements NodeLookup { public Mirror getMirror() { return mirror; } - @Override - public boolean isReady() { - return mirror != null && mirror.ready(); - } - - @Override public boolean updateCluster(ContentCluster cluster, NodeAddedOrRemovedListener listener) { if (mirror == null) return false; int mirrorVersion = mirror.updates(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java index 7f664e65dc0..9c0a94309a5 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java @@ -10,7 +10,6 @@ import org.junit.Test; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; @@ -66,22 +65,11 @@ public class DatabaseHandlerTest { } DatabaseHandler createHandler() throws Exception { - var handler = new DatabaseHandler(mockDbFactory, mockTimer, databaseAddress, 0, monitor); - handler.setConnectionEstablishmentIsAllowed(true); - return handler; + return new DatabaseHandler(mockDbFactory, mockTimer, databaseAddress, 0, monitor); } } @Test - public void can_not_connect_to_database_if_connectivity_is_not_allowed() throws Exception { - Fixture f = new Fixture(); - DatabaseHandler handler = f.createHandler(); - handler.setConnectionEstablishmentIsAllowed(false); - handler.doNextZooKeeperTask(f.createMockContext()); - assertTrue(handler.isClosed()); // No connectivity allowed yet - } - - @Test public void can_store_latest_cluster_state_bundle() throws Exception { Fixture f = new Fixture(); DatabaseHandler handler = f.createHandler(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java index d7bca47026f..b322c62967a 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java @@ -154,8 +154,4 @@ public class DummyCommunicator implements Communicator, NodeLookup { return false; } - @Override - public boolean isReady() { - return true; - } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java index 65199aa9957..46c57019b46 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.clustercontroller.core.hostinfo.ResourceUsage; import java.util.Arrays; import java.util.LinkedHashSet; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -78,11 +79,11 @@ public class FeedBlockUtil { static String createResourceUsageJson(Set<UsageDetails> usages) { // We deal only in the finest of manual JSON string building technologies(tm). String usageInnerJson = usages.stream() - .map(u -> String.format("\"%s\":{\"usage\": %.3g%s}", + .map(u -> String.format(Locale.US, "\"%s\":{\"usage\": %.3g%s}", u.type, u.usage, - (u.name != null ? String.format(",\"name\":\"%s\"", u.name) : ""))) + (u.name != null ? String.format(Locale.US, ",\"name\":\"%s\"", u.name) : ""))) .collect(Collectors.joining(",")); - return String.format("{\"content-node\":{\"resource-usage\":{%s}}}", usageInnerJson); + return String.format(Locale.US, "{\"content-node\":{\"resource-usage\":{%s}}}", usageInnerJson); } static NodeResourceExhaustion exhaustion(int index, String type) { diff --git a/config-application-package/src/main/java/com/yahoo/config/application/FileSystemWrapper.java b/config-application-package/src/main/java/com/yahoo/config/application/FileSystemWrapper.java index 8a08c56b3c0..0751d611ced 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/FileSystemWrapper.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/FileSystemWrapper.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; import com.yahoo.yolean.function.ThrowingFunction; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java index a1fae9bb148..20fc911241b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; public class EndpointCertificateMetadata { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java index 66a278b3fb6..d41b08cda5a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; import java.util.Optional; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index aa2a43e0d60..a7988b0ed70 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -272,10 +272,6 @@ public class VespaMetricSet { metrics.add(new Metric("search_connections.sum")); metrics.add(new Metric("search_connections.count")); metrics.add(new Metric("search_connections.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("active_queries.max")); - metrics.add(new Metric("active_queries.sum")); - metrics.add(new Metric("active_queries.count")); - metrics.add(new Metric("active_queries.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("feed.latency.max")); metrics.add(new Metric("feed.latency.sum")); metrics.add(new Metric("feed.latency.count")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java index 314ef9cc5a5..2b2b6593dd3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring.builder.xml; import com.yahoo.vespa.model.admin.monitoring.CloudWatch; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java index 025b7875677..a0835ab9b3f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.ConfigModelContext; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java index 909a29fd0a3..2bd343fc230 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java @@ -97,10 +97,10 @@ public class IndexedHierarchicDistributionValidator { } if (totalReadyCopies % groupCount != 0) { throw new IllegalArgumentException(getErrorMsgPrefix(clusterName) + "Expected equal amount of ready copies per group, but " + - totalReadyCopies + " ready copies is specified with " + groupCount + " groups"); + totalReadyCopies + " ready copies is specified with " + groupCount + " groups"); } if (totalReadyCopies == 0) { - System.err.println(getErrorMsgPrefix(clusterName) + "Warning. No ready copies configured. At least one is recommended."); + throw new IllegalArgumentException(getErrorMsgPrefix(clusterName) + "Warning. No ready copies configured. At least one is recommended."); } } diff --git a/config-model/src/test/derived/hnsw_index/test.sd b/config-model/src/test/derived/hnsw_index/test.sd index 395a852ec28..6006ea7a4f9 100644 --- a/config-model/src/test/derived/hnsw_index/test.sd +++ b/config-model/src/test/derived/hnsw_index/test.sd @@ -1,3 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search test { document test { field t1 type tensor(x[128]) { diff --git a/config-model/src/test/derived/language/language.sd b/config-model/src/test/derived/language/language.sd index a859438d885..54c3d83873a 100644 --- a/config-model/src/test/derived/language/language.sd +++ b/config-model/src/test/derived/language/language.sd @@ -1,3 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. schema language { document language { field language type string { @@ -16,4 +17,4 @@ schema language { stemming: best } -}
\ No newline at end of file +} diff --git a/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml b/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml index b8140f34617..cd4bcbb9e0d 100644 --- a/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml +++ b/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml @@ -1 +1,2 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="default" type="root" /> diff --git a/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml b/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml index 895e0663181..deaab98c606 100644 --- a/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml +++ b/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile-type id="root" inherits="native"> <field name="ranking.features.query(q_vec)" type="tensor<float>(x[5])" /> </query-profile-type> diff --git a/config-model/src/test/derived/nearestneighbor/test.sd b/config-model/src/test/derived/nearestneighbor/test.sd index ab5f6d85448..212fa684c01 100644 --- a/config-model/src/test/derived/nearestneighbor/test.sd +++ b/config-model/src/test/derived/nearestneighbor/test.sd @@ -1,3 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search test { document test { field id type int { diff --git a/config-model/src/test/derived/rankprofileinheritance/child.sd b/config-model/src/test/derived/rankprofileinheritance/child.sd index f76aa3a1f10..039963c3034 100644 --- a/config-model/src/test/derived/rankprofileinheritance/child.sd +++ b/config-model/src/test/derived/rankprofileinheritance/child.sd @@ -1,3 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. schema child { document child inherits parent1, parent2 { @@ -34,4 +35,4 @@ schema child { } -}
\ No newline at end of file +} diff --git a/config-model/src/test/derived/rankprofileinheritance/parent1.sd b/config-model/src/test/derived/rankprofileinheritance/parent1.sd index ea11ffbc82e..707de5c9f58 100644 --- a/config-model/src/test/derived/rankprofileinheritance/parent1.sd +++ b/config-model/src/test/derived/rankprofileinheritance/parent1.sd @@ -1,3 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. schema parent1 { document parent1 { diff --git a/config-model/src/test/derived/rankprofileinheritance/parent2.sd b/config-model/src/test/derived/rankprofileinheritance/parent2.sd index 1246f7264b3..b31144beaf7 100644 --- a/config-model/src/test/derived/rankprofileinheritance/parent2.sd +++ b/config-model/src/test/derived/rankprofileinheritance/parent2.sd @@ -1,3 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. schema parent2 { document parent2 { diff --git a/config-model/src/test/derived/tensor2/first.sd b/config-model/src/test/derived/tensor2/first.sd index 80554572503..f959b5a13c2 100644 --- a/config-model/src/test/derived/tensor2/first.sd +++ b/config-model/src/test/derived/tensor2/first.sd @@ -1,3 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search first { document first { field first_field type tensor(first[10]) { diff --git a/config-model/src/test/derived/tensor2/second.sd b/config-model/src/test/derived/tensor2/second.sd index ace0540c8bd..ec3af04aec9 100644 --- a/config-model/src/test/derived/tensor2/second.sd +++ b/config-model/src/test/derived/tensor2/second.sd @@ -1,3 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search second { document second { field second_field type tensor(second[10]) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java index d185362d362..51ebb17e218 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.core.ConsumersConfig; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MonitoringElementTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MonitoringElementTest.java index 614973459de..a4d2793dbbd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MonitoringElementTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MonitoringElementTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.core.MonitoringConfig; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java index 9a904592744..a62626e51e8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.telegraf.Telegraf; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidatorTest.java index 8fad2aa9af4..21a319ab29e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterValidatorTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation;// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import com.yahoo.config.model.MapConfigModelRegistry; @@ -58,4 +59,4 @@ public class AccessControlFilterValidatorTest { .applicationPackage(new MockApplicationPackage.Builder().withServices(SERVICES_XML).build()) .build(); } -}
\ No newline at end of file +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java index 40b8223479d..6c31150c973 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java index f3d199fc45c..3b7fded5f58 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import com.yahoo.config.application.api.ApplicationPackage; @@ -106,4 +107,4 @@ public class UriBindingsValidatorTest { "</services>"); } -}
\ No newline at end of file +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/default.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/default.xml index 95ee7d3d484..3cebc84a042 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/default.xml +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/default.xml @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="default"> <dimensions>d1</dimensions> <field name="a.b"><ref>main</ref></field> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/inherited.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/inherited.xml index 1c1102ab5db..819c9774a02 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/inherited.xml +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/inherited.xml @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="inherited"> <dimensions>d2</dimensions> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/main.xml b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/main.xml index efad3c8dd5c..464d9261307 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/main.xml +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants/main.xml @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="main"> <dimensions>d1</dimensions> <query-profile for="d1v" inherits="inherited" /> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java index 80ab6745b79..3be592e54e7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java @@ -254,10 +254,4 @@ public class IndexedHierarchicDistributionTest { getTwoGroupsCluster(4, 2, "2|*"); } - @Test - public void allowNoReadyCopies() throws Exception { - // The active one should be indexed anyhow. Setting up no ready copies - getTwoGroupsCluster(4, 0, "2|*"); - } - } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java index f0e0a6b8481..bebe237bab7 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; import com.yahoo.config.subscription.ConfigSourceSet; diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 037fd8d04e2..4dcc83a1c80 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -56,8 +56,9 @@ keepUnusedFileReferencesMinutes int default=300 # Bootstrapping # How long bootstrapping can take before giving up (in seconds) maxDurationOfBootstrap long default=7200 -# How long to sleep before redeploying again if it fails (in seconds) -sleepTimeWhenRedeployingFails long default=30 +# Initial time for how long to sleep before redeploying again if it fails (in seconds) +# Code uses backoff, so wait time will increase for every iteration +sleepTimeWhenRedeployingFails long default=15 # Features (to be overridden in configserver-config.xml if needed) buildMinimalSetOfConfigModels bool default=true diff --git a/configdefinitions/src/vespa/reindexing.def b/configdefinitions/src/vespa/reindexing.def index 6eb084f6417..ef5174f9bde 100644 --- a/configdefinitions/src/vespa/reindexing.def +++ b/configdefinitions/src/vespa/reindexing.def @@ -1,4 +1,4 @@ -# Copyright 2020 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Reindexing status per document type, for a Vespa application namespace=vespa.config.content.reindexing diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index be26e880440..24926f51b15 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -204,23 +204,23 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private boolean redeployAllApplications() throws InterruptedException { Instant end = Instant.now().plus(maxDurationOfRedeployment); - List<ApplicationId> applicationsNotRedeployed = applicationRepository.listApplications(); - Collections.shuffle(applicationsNotRedeployed); + List<ApplicationId> applicationsToRedeploy = applicationRepository.listApplications(); + Collections.shuffle(applicationsToRedeploy); long failCount = 0; do { - applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed); - if ( ! applicationsNotRedeployed.isEmpty() && ! sleepTimeWhenRedeployingFails.isZero()) { + applicationsToRedeploy = redeployApplications(applicationsToRedeploy); + if ( ! applicationsToRedeploy.isEmpty() && ! sleepTimeWhenRedeployingFails.isZero()) { Duration sleepTime = sleepTimeWhenRedeployingFails.multipliedBy(++failCount); if (sleepTime.compareTo(Duration.ofMinutes(10)) > 0) sleepTime = Duration.ofMinutes(10); - log.log(Level.INFO, "Redeployment of " + applicationsNotRedeployed + " not finished, will retry in " + sleepTime); + log.log(Level.INFO, "Redeployment of " + applicationsToRedeploy + " not finished, will retry in " + sleepTime); Thread.sleep(sleepTime.toMillis()); } - } while ( ! applicationsNotRedeployed.isEmpty() && Instant.now().isBefore(end)); + } while ( ! applicationsToRedeploy.isEmpty() && Instant.now().isBefore(end)); - if ( ! applicationsNotRedeployed.isEmpty()) { + if ( ! applicationsToRedeploy.isEmpty()) { log.log(Level.SEVERE, "Redeploying applications not finished after " + maxDurationOfRedeployment + - ", exiting, applications that failed redeployment: " + applicationsNotRedeployed); + ", exiting, applications that failed redeployment: " + applicationsToRedeploy); return false; } return true; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java index 4ed93f0aa64..289a03db3e7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.cloud.config.ConfigserverConfig; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java index a0ad87a39c9..7f95d7e30d0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.ApplicationId; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 7ba8b1be491..5519ffc1bdc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.maintenance; import com.yahoo.cloud.config.ConfigserverConfig; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java index 2c63eec0ce9..d6feecb7479 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.metrics; import ai.vespa.util.http.VespaHttpClientBuilder; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsAggregator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsAggregator.java index e1a0c2dc253..025195066b7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsAggregator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsAggregator.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.metrics; import com.yahoo.slime.Inspector; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java index 91295ca8bee..ae70225e8a4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.metrics; import com.yahoo.config.model.api.HostInfo; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java index 13c88ece6d7..afe57b75d61 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; import com.yahoo.config.model.api.EndpointCertificateMetadata; diff --git a/configserver/src/test/apps/app-major-version-2/deployment.xml b/configserver/src/test/apps/app-major-version-2/deployment.xml index 7523c104b7e..1a3b4b35e9c 100644 --- a/configserver/src/test/apps/app-major-version-2/deployment.xml +++ b/configserver/src/test/apps/app-major-version-2/deployment.xml @@ -1 +1,2 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <deployment version='1.0' major-version='2'/> diff --git a/configserver/src/test/apps/app-with-multiple-clusters/services.xml b/configserver/src/test/apps/app-with-multiple-clusters/services.xml index 735bd04b2f9..3daf39e34aa 100644 --- a/configserver/src/test/apps/app-with-multiple-clusters/services.xml +++ b/configserver/src/test/apps/app-with-multiple-clusters/services.xml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2020 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services version="1.0"> <content id="foo" version="1.0"> diff --git a/configserver/src/test/apps/zkfeed/configdefinitions/a.b.a.b.test2.def b/configserver/src/test/apps/zkfeed/configdefinitions/a.b.a.b.test2.def index 9e5c787df09..d8a34c6dd66 100644 --- a/configserver/src/test/apps/zkfeed/configdefinitions/a.b.a.b.test2.def +++ b/configserver/src/test/apps/zkfeed/configdefinitions/a.b.a.b.test2.def @@ -1,3 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=a.b test2 string default="" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/DefaultClusterReindexingStatusClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/DefaultClusterReindexingStatusClientTest.java index 7b8d5d9cc07..a8f35325b57 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/DefaultClusterReindexingStatusClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/DefaultClusterReindexingStatusClientTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import com.github.tomakehurst.wiremock.junit.WireMockRule; @@ -124,4 +125,4 @@ public class DefaultClusterReindexingStatusClientTest { } -}
\ No newline at end of file +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SecretStoreValidatorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SecretStoreValidatorTest.java index 8e5dd16bc05..ed668bfef15 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SecretStoreValidatorTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SecretStoreValidatorTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; import com.github.tomakehurst.wiremock.junit.WireMockRule; @@ -79,4 +80,4 @@ public class SecretStoreValidatorTest { List.of(new ServiceInfo("default", "container", null, null, "", "localhost")) )); } -}
\ No newline at end of file +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetrieverTest.java index 5aada44b573..0aad2f1ee26 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetrieverTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.metrics; import com.yahoo.config.ConfigInstance; diff --git a/container-core-config/README.md b/container-core-config/README.md index 5bb4c3860e1..ac37c242a5f 100644 --- a/container-core-config/README.md +++ b/container-core-config/README.md @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # container-core-config Contains config definitions with package `com.yahoo.container.core` that are used by other modules. diff --git a/container-core/src/main/java/com/yahoo/container/core/HandlerMetricContextUtil.java b/container-core/src/main/java/com/yahoo/container/core/HandlerMetricContextUtil.java index cfb48339dbe..b46cca17848 100644 --- a/container-core/src/main/java/com/yahoo/container/core/HandlerMetricContextUtil.java +++ b/container-core/src/main/java/com/yahoo/container/core/HandlerMetricContextUtil.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.core; import com.yahoo.jdisc.Metric; diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleStarter.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleStarter.java index 4a87c27b990..416bb93d9e2 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/BundleStarter.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/BundleStarter.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.core.config; import org.osgi.framework.Bundle; diff --git a/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java index 3951e656736..4f7f718345e 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/PlatformBundleLoader.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.core.config; import com.yahoo.osgi.Osgi; diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java index 33c6fbefa71..5e3ec8f2f05 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.metrics; import ai.vespa.util.http.VespaHttpClientBuilder; diff --git a/container-core/src/main/java/com/yahoo/restapi/Path.java b/container-core/src/main/java/com/yahoo/restapi/Path.java index 23a791e7532..9fc924adbe2 100644 --- a/container-core/src/main/java/com/yahoo/restapi/Path.java +++ b/container-core/src/main/java/com/yahoo/restapi/Path.java @@ -46,6 +46,7 @@ public class Path { this(uri, ""); } + // TODO (freva): Remove, used by factory public Path(URI uri, String optionalPrefix) { this.optionalPrefix = optionalPrefix; this.pathString = uri.getRawPath(); diff --git a/container-core/src/test/java/com/yahoo/container/core/config/PlatformBundleLoaderTest.java b/container-core/src/test/java/com/yahoo/container/core/config/PlatformBundleLoaderTest.java index 931b0c547fd..3111841439e 100644 --- a/container-core/src/test/java/com/yahoo/container/core/config/PlatformBundleLoaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/core/config/PlatformBundleLoaderTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.core.config; import com.yahoo.osgi.Osgi; diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/MockSnapshotProvider.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/MockSnapshotProvider.java index 5cc65be855d..0895aad83ab 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/MockSnapshotProvider.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/MockSnapshotProvider.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.state; import java.io.PrintStream; diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 8fc31627800..941015b2fae 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -57,7 +57,6 @@ public class StatisticsSearcher extends Searcher { private static final String QUERY_LATENCY_METRIC = "query_latency"; private static final String QUERY_HIT_OFFSET_METRIC = "query_hit_offset"; private static final String QUERIES_METRIC = "queries"; - private static final String ACTIVE_QUERIES_METRIC = "active_queries"; private static final String PEAK_QPS_METRIC = "peak_qps"; private static final String DOCS_COVERED_METRIC = "documents_covered"; private static final String DOCS_TOTAL_METRIC = "documents_total"; @@ -84,12 +83,12 @@ public class StatisticsSearcher extends Searcher { // Naming of enums are reflected directly in metric dimensions and should not be changed as they are public API private enum DegradedReason { match_phase, adaptive_timeout, timeout, non_ideal_state } - private Metric metric; - private Map<String, Metric.Context> chainContexts = new CopyOnWriteHashMap<>(); - private Map<String, Metric.Context> statePageOnlyContexts = new CopyOnWriteHashMap<>(); - private Map<String, Map<DegradedReason, Metric.Context>> degradedReasonContexts = new CopyOnWriteHashMap<>(); - private Map<String, Map<String, Metric.Context>> relevanceContexts = new CopyOnWriteHashMap<>(); - private java.util.Timer scheduler = new java.util.Timer(true); + private final Metric metric; + private final Map<String, Metric.Context> chainContexts = new CopyOnWriteHashMap<>(); + private final Map<String, Metric.Context> statePageOnlyContexts = new CopyOnWriteHashMap<>(); + private final Map<String, Map<DegradedReason, Metric.Context>> degradedReasonContexts = new CopyOnWriteHashMap<>(); + private final Map<String, Map<String, Metric.Context>> relevanceContexts = new CopyOnWriteHashMap<>(); + private final java.util.Timer scheduler = new java.util.Timer(true); private class PeakQpsReporter extends java.util.TimerTask { private long prevMaxQPSTime = System.currentTimeMillis(); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java index ac8f0a59c20..7d9b3ca1034 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; import com.yahoo.search.cluster.ClusterMonitor; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java index c0579b5d36e..1b39f14fd86 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.searchcluster; import com.yahoo.prelude.Pong; diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java index d40e1cfc4a6..3a392fcbda4 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.querytransform; import com.yahoo.prelude.query.*; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr index c81b0e1819e..728c494682b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr @@ -1,3 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. or1 +> ?or2 ?or3; equiv1 +> =equiv2 =equiv3; diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java index 39d4fec2716..636298e8669 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.compiled; import com.yahoo.search.query.profile.config.QueryProfilesConfig; diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/inheritance/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/inheritance/parent.xml index b3443fab646..47873fcce4f 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/inheritance/parent.xml +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/inheritance/parent.xml @@ -1,4 +1,4 @@ -<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="parent"> <field name="a.b">a.b-parent</field> <field name="a.b.c">a.b.c-parent</field> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml index 000fd3e1c5b..d75961fb584 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml @@ -1,2 +1,3 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="profile1" type="type1"> </query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml index f6539da23e8..e08362043d2 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml @@ -1,2 +1,3 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="profile2" type="type2"> </query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml index 3dfaab9c5f2..33a223d026b 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile-type id="type1"> <field name="ranking.features.query(tensor_1)" type="tensor<float>(x[1])" /> </query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml index ed7cf23e464..a856ddddb91 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile-type id="type2"> <field name="ranking.features.query(tensor_2)" type="tensor<float>(x[2])" /> <field name="ranking.features.query(tensor_3)" type="tensor<float>(x[3])" /> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/default.xml index 95ee7d3d484..3cebc84a042 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/default.xml +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/default.xml @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="default"> <dimensions>d1</dimensions> <field name="a.b"><ref>main</ref></field> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/inherited.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/inherited.xml index 1c1102ab5db..819c9774a02 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/inherited.xml +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/inherited.xml @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="inherited"> <dimensions>d2</dimensions> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/main.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/main.xml index efad3c8dd5c..464d9261307 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/main.xml +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/variants/main.xml @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="main"> <dimensions>d1</dimensions> <query-profile for="d1v" inherits="inherited" /> diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java index 1aa61d7535b..a72fbeeaac9 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.querytransform; import com.yahoo.component.chain.Chain; diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 3f0000367e7..5c97c92e89a 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -123,6 +123,12 @@ public class YqlParserTestCase { } @Test + public void testHitLimit() { + assertParse("select artist_name, track_name, track_uri from sources * where (myField contains ([{\"prefix\":true}]\"m\") and ([{\"hitLimit\": 5000, \"descending\": true}]range(static_score,0,Infinity))) limit 30 offset 0;", + "AND myField:m* static_score:[0;;-5000]"); + } + + @Test public void test() { assertParse("select foo from bar where title contains \"madonna\";", "title:madonna"); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java deleted file mode 100644 index 255684d0b66..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.application.v4.model; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.yahoo.component.Version; - -import java.util.Optional; - -/** - * @author gjoranv - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class DeployOptions { - - public final boolean deployDirectly; - public final Optional<String> vespaVersion; - public final boolean ignoreValidationErrors; - public final boolean deployCurrentVersion; - - @JsonCreator - public DeployOptions(@JsonProperty("deployDirectly") boolean deployDirectly, - @JsonProperty("vespaVersion") Optional<Version> vespaVersion, - @JsonProperty("ignoreValidationErrors") boolean ignoreValidationErrors, - @JsonProperty("deployCurrentVersion") boolean deployCurrentVersion) { - this.deployDirectly = deployDirectly; - this.vespaVersion = vespaVersion.map(Version::toString); - this.ignoreValidationErrors = ignoreValidationErrors; - this.deployCurrentVersion = deployCurrentVersion; - } - - @Override - public String toString() { - return "DeployData{" + - "deployDirectly=" + deployDirectly + - ", vespaVersion=" + vespaVersion.orElse("None") + - ", ignoreValidationErrors=" + ignoreValidationErrors + - ", deployCurrentVersion=" + deployCurrentVersion + - '}'; - } - - public static DeployOptions none() { - return new DeployOptions(false, Optional.empty(), false, false); - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index 4a577632746..d0b9653bbf3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.application.v4.model; import com.yahoo.component.Version; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ProtonMetrics.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ProtonMetrics.java index ed88902e094..2d9116e18d6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ProtonMetrics.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ProtonMetrics.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.application.v4.model; import com.fasterxml.jackson.databind.JsonNode; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java index c7384bf85d8..cdde20b4554 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.archive; import com.yahoo.config.provision.TenantName; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java index 6859c7a8645..66f753a6186 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.archive; import com.yahoo.config.provision.TenantName; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionMethod.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionMethod.java index fd586b0faf0..5cf753bd312 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionMethod.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionMethod.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.billing; public enum CollectionMethod { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/QuotaCalculator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/QuotaCalculator.java index 69375969d58..a7d5452b221 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/QuotaCalculator.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/QuotaCalculator.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.billing; /** diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java index 0bf3f0440d2..5216c1421fa 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.certificates; import java.util.List; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidator.java index 2c56cbf580d..acf775315ec 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidator.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidator.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.certificates; import com.yahoo.config.provision.zone.ZoneId; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java index 90cbaecb628..ea92f84f72f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.certificates; import com.yahoo.config.provision.zone.ZoneId; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorMock.java index fb0121d3612..7e89a3becd5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorMock.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.certificates; import com.yahoo.config.provision.zone.ZoneId; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ProxyResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ProxyResponse.java index 8570155861a..b1676901441 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ProxyResponse.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ProxyResponse.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.container.jdisc.HttpResponse; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/QuotaUsage.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/QuotaUsage.java index 2c1efd2109b..b6e79ad2635 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/QuotaUsage.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/QuotaUsage.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchivePatch.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchivePatch.java index b3ba4e1ba4f..0d22b5e050a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchivePatch.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchivePatch.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.noderepository; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/SystemMonitor.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/SystemMonitor.java index b8b5400f7b4..6e1c71e2be7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/SystemMonitor.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/SystemMonitor.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.organization; import com.yahoo.component.Version; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java index 633a75d070c..5bb61979f1d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; import com.yahoo.config.provision.ApplicationId; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/DummySystemMonitor.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/DummySystemMonitor.java index e2e5fe9e155..d13eb897d0f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/DummySystemMonitor.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/DummySystemMonitor.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.stubs; import com.yahoo.component.Version; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 72210ec26ed..a0da8cbddba 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -6,7 +6,6 @@ import com.yahoo.restapi.Path; import java.net.URI; import java.util.EnumSet; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -21,13 +20,11 @@ import java.util.Set; enum PathGroup { /** Paths exclusive to operators (including read), used for system management. */ - classifiedOperator(PathPrefix.api, - "/configserver/v1/{*}", + classifiedOperator("/configserver/v1/{*}", "/deployment/v1/{*}"), /** Paths used for system management by operators. */ - operator(PathPrefix.none, - "/controller/v1/{*}", + operator("/controller/v1/{*}", "/flags/v1/{*}", "/loadbalancers/v1/{*}", "/nodes/v2/{*}", @@ -42,61 +39,51 @@ enum PathGroup { "/changemanagement/v1/{*}"), /** Paths used for creating and reading user resources. */ - user(PathPrefix.api, - "/application/v4/user", + user("/application/v4/user", "/athenz/v1/{*}"), /** Paths used for creating tenants with proper access control. */ tenant(Matcher.tenant, - PathPrefix.api, "/application/v4/tenant/{tenant}"), /** Paths used for user management on the tenant level. */ tenantUsers(Matcher.tenant, - PathPrefix.api, "/user/v1/tenant/{tenant}"), /** Paths used by tenant administrators. */ tenantInfo(Matcher.tenant, - PathPrefix.api, "/application/v4/tenant/{tenant}/application/", "/application/v4/tenant/{tenant}/info/", "/routing/v1/status/tenant/{tenant}/{*}"), tenantKeys(Matcher.tenant, - PathPrefix.api, "/application/v4/tenant/{tenant}/key/"), + tenantArchiveAccess(Matcher.tenant, + "/application/v4/tenant/{tenant}/archive-access"), billingToken(Matcher.tenant, - PathPrefix.api, "/billing/v1/tenant/{tenant}/token"), billingInstrument(Matcher.tenant, - PathPrefix.api, "/billing/v1/tenant/{tenant}/instrument/{*}"), billingPlan(Matcher.tenant, - PathPrefix.api, "/billing/v1/tenant/{tenant}/plan/{*}"), billingCollection(Matcher.tenant, - PathPrefix.api, "/billing/v1/tenant/{tenant}/collection/{*}"), billingList(Matcher.tenant, - PathPrefix.api, "/billing/v1/tenant/{tenant}/billing/{*}"), applicationKeys(Matcher.tenant, Matcher.application, - PathPrefix.api, "/application/v4/tenant/{tenant}/application/{application}/key/"), /** Path for the base application resource. */ application(Matcher.tenant, Matcher.application, - PathPrefix.api, "/application/v4/tenant/{tenant}/application/{application}", "/application/v4/tenant/{tenant}/application/{application}/instance/", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}"), @@ -104,13 +91,11 @@ enum PathGroup { /** Paths used for user management on the application level. */ applicationUsers(Matcher.tenant, Matcher.application, - PathPrefix.api, "/user/v1/tenant/{tenant}/application/{application}"), /** Paths used by application administrators. */ applicationInfo(Matcher.tenant, Matcher.application, - PathPrefix.api, "/application/v4/tenant/{tenant}/application/{application}/package", "/application/v4/tenant/{tenant}/application/{application}/compile-version", "/application/v4/tenant/{tenant}/application/{application}/deployment", @@ -139,7 +124,6 @@ enum PathGroup { developmentRestart(Matcher.tenant, Matcher.application, Matcher.instance, - PathPrefix.api, "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/dev/region/{region}/restart", "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/perf/region/{region}/restart", "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{instance}/restart", @@ -149,7 +133,6 @@ enum PathGroup { /** Path used to restart production nodes. */ productionRestart(Matcher.tenant, Matcher.application, - PathPrefix.api, "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}/restart", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/test/region/{region}/restart", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/staging/region/{region}/restart", @@ -160,7 +143,6 @@ enum PathGroup { /** Path used to manipulate reindexing status. */ reindexing(Matcher.tenant, Matcher.application, - PathPrefix.api, "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/reindex", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/{environment}/region/{region}/reindexing"), @@ -168,7 +150,6 @@ enum PathGroup { developmentDeployment(Matcher.tenant, Matcher.application, Matcher.instance, - PathPrefix.api, "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{job}", "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/dev/region/{region}", "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/dev/region/{region}/deploy", @@ -185,7 +166,6 @@ enum PathGroup { /** Paths used for production deployments. */ productionDeployment(Matcher.tenant, Matcher.application, - PathPrefix.api, "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}/deploy", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/test/region/{region}", @@ -202,84 +182,76 @@ enum PathGroup { /** Paths used for continuous deployment to production. */ submission(Matcher.tenant, Matcher.application, - PathPrefix.api, "/application/v4/tenant/{tenant}/application/{application}/submit", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/submit"), /** Paths used for other tasks by build services. */ // TODO: This will vanish. buildService(Matcher.tenant, Matcher.application, - PathPrefix.api, "/application/v4/tenant/{tenant}/application/{application}/jobreport", "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/jobreport"), /** Paths which contain (not very strictly) classified information about customers. */ - classifiedTenantInfo(PathPrefix.api, - "/application/v4/", + classifiedTenantInfo("/application/v4/", "/application/v4/tenant/"), /** Paths which contain (not very strictly) classified information about, e.g., customers. */ - classifiedInfo(PathPrefix.none, - "/", + classifiedInfo("/", "/d/{*}"), /** Paths providing public information. */ - publicInfo(PathPrefix.api, - "/user/v1/user", // Information about who you are. + publicInfo("/user/v1/user", // Information about who you are. "/badge/v1/{*}", // Badges for deployment jobs. "/zone/v1/{*}"), // Lists environment and regions. /** Paths used for deploying system-wide feature flags. */ - systemFlagsDeploy(PathPrefix.none, "/system-flags/v1/deploy"), + systemFlagsDeploy("/system-flags/v1/deploy"), /** Paths used for "dry-running" system-wide feature flags. */ - systemFlagsDryrun(PathPrefix.none, "/system-flags/v1/dryrun"), + systemFlagsDryrun("/system-flags/v1/dryrun"), /** Paths used for receiving payment callbacks */ - paymentProcessor(PathPrefix.none, "/payment/notification"), + paymentProcessor("/payment/notification"), /** Paths used for invoice management */ - hostedAccountant(PathPrefix.api, - "/billing/v1/invoice/{*}", - "/billing/v1/billing"), + hostedAccountant("/billing/v1/invoice/{*}", + "/billing/v1/billing"), /** Path used for listing endpoint certificate request info */ - endpointCertificateRequestInfo(PathPrefix.none, "/certificateRequests/"), + endpointCertificateRequestInfo("/certificateRequests/"), /** Path used for secret store management */ - secretStore(Matcher.tenant, PathPrefix.api, "/application/v4/tenant/{tenant}/secret-store/{*}"); + secretStore(Matcher.tenant, "/application/v4/tenant/{tenant}/secret-store/{*}"); final List<String> pathSpecs; - final PathPrefix prefix; final List<Matcher> matchers; - PathGroup(PathPrefix prefix, String... pathSpecs) { - this(List.of(), prefix, List.of(pathSpecs)); + PathGroup(String... pathSpecs) { + this(List.of(), List.of(pathSpecs)); } - PathGroup(Matcher first, PathPrefix prefix, String... pathSpecs) { - this(List.of(first), prefix, List.of(pathSpecs)); + PathGroup(Matcher first, String... pathSpecs) { + this(List.of(first), List.of(pathSpecs)); } - PathGroup(Matcher first, Matcher second, PathPrefix prefix, String... pathSpecs) { - this(List.of(first, second), prefix, List.of(pathSpecs)); + PathGroup(Matcher first, Matcher second, String... pathSpecs) { + this(List.of(first, second), List.of(pathSpecs)); } - PathGroup(Matcher first, Matcher second, Matcher third, PathPrefix prefix, String... pathSpecs) { - this(List.of(first, second, third), prefix, List.of(pathSpecs)); + PathGroup(Matcher first, Matcher second, Matcher third, String... pathSpecs) { + this(List.of(first, second, third), List.of(pathSpecs)); } /** Creates a new path group, if the given context matchers are each present exactly once in each of the given specs. */ - PathGroup(List<Matcher> matchers, PathPrefix prefix, List<String> pathSpecs) { + PathGroup(List<Matcher> matchers, List<String> pathSpecs) { this.matchers = matchers; - this.prefix = prefix; this.pathSpecs = pathSpecs; } /** Returns path if it matches any spec in this group, with match groups set by the match. */ private Optional<Path> get(URI uri) { - Path matcher = new Path(uri, prefix.prefix); + Path matcher = new Path(uri); for (String spec : pathSpecs) // Iterate to be sure the Path's state is that of the match. if (matcher.matches(spec)) return Optional.of(matcher); return Optional.empty(); @@ -352,21 +324,4 @@ enum PathGroup { } - /** - * The valid prefixes of paths in a {@link PathGroup}. Provides flexibility in cases where paths are made available - * under a non-root path. - */ - enum PathPrefix { - - none(""), - api("/api"); - - private final String prefix; - - PathPrefix(String prefix) { - this.prefix = Objects.requireNonNull(prefix); - } - - } - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index ad739d16ff8..b48e786c178 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -72,6 +72,11 @@ enum Policy { .on(PathGroup.tenant, PathGroup.tenantInfo, PathGroup.tenantUsers, PathGroup.applicationUsers) .in(SystemName.all())), + /** Access to set and unset archive access role under a tenant. */ + tenantArchiveAccessManagement(Privilege.grant(Action.update, Action.delete) + .on(PathGroup.tenantArchiveAccess) + .in(SystemName.all())), + /** Access to create application under a certain tenant. */ applicationCreate(Privilege.grant(Action.create) .on(PathGroup.application) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index aeb5419b682..a0ee0fe3548 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -63,6 +63,7 @@ public enum RoleDefinition { administrator(Policy.tenantUpdate, Policy.tenantManager, Policy.tenantDelete, + Policy.tenantArchiveAccessManagement, Policy.applicationManager, Policy.keyRevokal, Policy.paymentInstrumentRead, diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeployOptionsTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeployOptionsTest.java deleted file mode 100644 index a97fde0a6d6..00000000000 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeployOptionsTest.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.identifiers; - -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; -import org.junit.Test; - -import java.io.IOException; -import java.util.Optional; - -import static org.junit.Assert.assertEquals; - -/** - * @author mortent - */ -public class DeployOptionsTest { - - @Test - public void it_serializes_version() throws IOException { - DeployOptions options = new DeployOptions(false, Optional.of(new Version("6.98.227")), false, false); - final ObjectMapper objectMapper = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .registerModule(new Jdk8Module()); - - String string = objectMapper.writeValueAsString(options); - assertEquals("{\"deployDirectly\":false,\"vespaVersion\":\"6.98.227\",\"ignoreValidationErrors\":false,\"deployCurrentVersion\":false}", string); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 0a1af8817d7..51b7a24b5e4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -22,9 +22,7 @@ import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.api.ActivateResult; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; @@ -39,12 +37,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerE import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; @@ -63,7 +59,6 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; -import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; @@ -348,14 +343,8 @@ public class ApplicationController { return application.withNewInstance(id.instance()); } - public ActivateResult deploy(ApplicationId applicationId, ZoneId zone, - Optional<ApplicationPackage> applicationPackageFromDeployer, - DeployOptions options) { - return deploy(applicationId, zone, applicationPackageFromDeployer, Optional.empty(), options); - } - /** Deploys an application package for an existing application instance. */ - public ActivateResult deploy2(JobId job, boolean deploySourceVersions) { // TODO jonmv: make it number one! + public ActivateResult deploy(JobId job, boolean deploySourceVersions) { if (job.application().instance().isTester()) throw new IllegalArgumentException("'" + job.application() + "' is a tester application!"); @@ -424,93 +413,6 @@ public class ApplicationController { : applicationStore.get(application.tenant(), application.application(), revision)); } - public ActivateResult deploy(ApplicationId instanceId, ZoneId zone, - Optional<ApplicationPackage> applicationPackageFromDeployer, - Optional<ApplicationVersion> applicationVersionFromDeployer, - DeployOptions options) { - if (instanceId.instance().isTester()) - throw new IllegalArgumentException("'" + instanceId + "' is a tester application!"); - - TenantAndApplicationId applicationId = TenantAndApplicationId.from(instanceId); - if (getInstance(instanceId).isEmpty()) - createInstance(instanceId); - - try (Lock deploymentLock = lockForDeployment(instanceId, zone)) { - Version platformVersion; - ApplicationVersion applicationVersion; - ApplicationPackage applicationPackage; - Set<ContainerEndpoint> endpoints; - Optional<EndpointCertificateMetadata> endpointCertificateMetadata; - - try (Lock lock = lock(applicationId)) { - LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); - InstanceName instance = instanceId.instance(); - - boolean manuallyDeployed = options.deployDirectly || zone.environment().isManuallyDeployed(); - boolean preferOldestVersion = options.deployCurrentVersion; - - // Determine versions to use. - if (manuallyDeployed) { - applicationVersion = applicationVersionFromDeployer.orElse(ApplicationVersion.unknown); - applicationPackage = applicationPackageFromDeployer.orElseThrow( - () -> new IllegalArgumentException("Application package must be given when deploying to " + zone)); - platformVersion = options.vespaVersion.map(Version::new) - .orElse(applicationPackage.deploymentSpec().majorVersion() - .flatMap(this::lastCompatibleVersion) - .orElseGet(controller::readSystemVersion)); - } - else { - JobType jobType = JobType.from(controller.system(), zone) - .orElseThrow(() -> new IllegalArgumentException("No job is known for " + zone + ".")); - var run = controller.jobController().last(instanceId, jobType); - if (run.map(Run::hasEnded).orElse(true)) - return unexpectedDeployment(instanceId, zone); - Versions versions = run.get().versions(); - platformVersion = preferOldestVersion ? versions.sourcePlatform().orElse(versions.targetPlatform()) - : versions.targetPlatform(); - applicationVersion = preferOldestVersion ? versions.sourceApplication().orElse(versions.targetApplication()) - : versions.targetApplication(); - - applicationPackage = getApplicationPackage(instanceId, applicationVersion); - applicationPackage = withTesterCertificate(applicationPackage, instanceId, jobType); - validateRun(application.get().require(instance), zone, platformVersion, applicationVersion); - } - - endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata( - application.get().require(instance), zone, applicationPackage.deploymentSpec().instance(instance)); - - endpoints = controller.routing().registerEndpointsInDns(application.get(), instance, zone); - } // Release application lock while doing the deployment, which is a lengthy task. - - // Carry out deployment without holding the application lock. - ActivateResult result = deploy(instanceId, applicationPackage, zone, platformVersion, - endpoints, endpointCertificateMetadata, Optional.empty()); - - // Record the quota usage for this application - var quotaUsage = deploymentQuotaUsage(zone, instanceId); - - lockApplicationOrThrow(applicationId, application -> - store(application.with(instanceId.instance(), - instance -> instance.withNewDeployment(zone, applicationVersion, platformVersion, - clock.instant(), warningsFrom(result), - quotaUsage)))); - return result; - } - } - - private ApplicationPackage withTesterCertificate(ApplicationPackage applicationPackage, ApplicationId id, JobType type) { - if (applicationPackage.trustedCertificates().isEmpty()) - return applicationPackage; - - // TODO jonmv: move this to the caller, when external build service is removed. - Run run = controller.jobController().last(id, type) - .orElseThrow(() -> new IllegalStateException("Last run of " + type + " for " + id + " not found")); - if (run.testerCertificate().isEmpty()) - return applicationPackage; - - return applicationPackage.withTrustedCertificate(run.testerCertificate().get()); - } - /** Fetches the requested application package from the artifact store(s). */ public ApplicationPackage getApplicationPackage(ApplicationId id, ApplicationVersion version) { return new ApplicationPackage(applicationStore.get(id.tenant(), id.application(), version)); @@ -616,18 +518,6 @@ public class ApplicationController { } } - private ActivateResult unexpectedDeployment(ApplicationId application, ZoneId zone) { - Log logEntry = new Log(); - logEntry.level = "WARNING"; - logEntry.time = clock.instant().toEpochMilli(); - logEntry.message = "Ignoring deployment of application '" + application + "' to " + zone + - " as a deployment is not currently expected"; - PrepareResponse prepareResponse = new PrepareResponse(); - prepareResponse.log = List.of(logEntry); - prepareResponse.configChangeActions = new ConfigChangeActions(List.of(), List.of(), List.of()); - return new ActivateResult(new RevisionId("0"), prepareResponse, 0); - } - private LockedApplication withoutDeletedDeployments(LockedApplication application, InstanceName instance) { DeploymentSpec deploymentSpec = application.get().deploymentSpec(); List<ZoneId> deploymentsToRemove = application.get().require(instance).productionDeployments().values().stream() @@ -844,17 +734,6 @@ public class ApplicationController { return curator.lockForDeployment(application, zone); } - /** Verify that we don't downgrade an existing production deployment. */ - private void validateRun(Instance instance, ZoneId zone, Version platformVersion, ApplicationVersion applicationVersion) { - Deployment deployment = instance.deployments().get(zone); - if ( zone.environment().isProduction() && deployment != null - && ( platformVersion.compareTo(deployment.version()) < 0 && ! instance.change().isPinned() - || applicationVersion.compareTo(deployment.applicationVersion()) < 0)) - throw new IllegalArgumentException(String.format("Rejecting deployment of application %s to %s, as the requested versions (platform: %s, application: %s)" + - " are older than the currently deployed (platform: %s, application: %s).", - instance.id(), zone, platformVersion, applicationVersion, deployment.version(), deployment.applicationVersion())); - } - /** * Verifies that the application can be deployed to the tenant, following these rules: * diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index c6046751696..4954628a46a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.google.common.collect.ImmutableMap; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index c994f0ae4fd..1a1b6988a96 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -65,8 +65,8 @@ public enum SystemApplication { .orElse(false); } - /** Returns whether this should receive OS upgrades in given zone */ - public boolean shouldUpgradeOsIn() { + /** Returns whether this should receive OS upgrades */ + public boolean shouldUpgradeOs() { return nodeType.isHost(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index f071fe86002..6f964999fba 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.certificate; import com.google.common.hash.Hashing; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index e1587d250ec..f2df1cce15b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -184,7 +184,7 @@ public class InternalStepRunner implements StepRunner { } private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, DualLogger logger) { - return deploy(() -> controller.applications().deploy2(id.job(), setTheStage), + return deploy(() -> controller.applications().deploy(id.job(), setTheStage), controller.jobController().run(id).get() .stepInfo(setTheStage ? deployInitialReal : deployReal).get() .startTime().get(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index f19c07e4650..65d3f666309 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -41,10 +41,8 @@ import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; -import java.util.Queue; import java.util.Set; import java.util.SortedMap; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -95,7 +93,7 @@ public class JobController { this.logs = new BufferedLogStore(curator, controller.serviceRegistry().runDataStore()); this.cloud = controller.serviceRegistry().testerCloud(); this.badges = new Badges(controller.zoneRegistry().badgeUrl()); - this.metric = new JobMetrics(controller.metric(), controller.system()); + this.metric = new JobMetrics(controller.metric(), controller::system); } public TesterCloud cloud() { return cloud; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index 80924c3c0aa..49da987ea18 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -6,6 +6,7 @@ import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import java.util.Map; +import java.util.function.Supplier; /** * Records metrics related to deployment jobs. @@ -25,9 +26,9 @@ public class JobMetrics { public static final String success = "deployment.success"; private final Metric metric; - private final SystemName system; + private final Supplier<SystemName> system; - public JobMetrics(Metric metric, SystemName system) { + public JobMetrics(Metric metric, Supplier<SystemName> system) { this.metric = metric; this.system = system; } @@ -45,7 +46,7 @@ public class JobMetrics { "tenantName", id.application().tenant().value(), "app", id.application().application().value() + "." + id.application().instance().value(), "test", Boolean.toString(id.type().isTest()), - "zone", id.type().zone(system).value()); + "zone", id.type().zone(system.get()).value()); } static String valueOf(RunStatus status) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java index 921bf045873..9326035d4c7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java index 42b442bf7b0..7d94a4c728f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationMetaDataGarbageCollector.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Controller; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java index 1bfdbc55f26..00e8c508695 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index a20043d35c0..43e9ce51040 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -62,7 +62,7 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { @Override protected boolean expectUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { return cloud.equals(zone.getCloudName()) && // Cloud is managed by this upgrader - application.shouldUpgradeOsIn() && // Application should upgrade in this cloud + application.shouldUpgradeOs() && // Application should upgrade in this cloud canUpgrade(node); // Node is in an upgradable state } @@ -76,7 +76,7 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> { @Override protected boolean changeTargetTo(OsVersionTarget target, SystemApplication application, ZoneApi zone) { - if (!application.shouldUpgradeOsIn()) return false; + if (!application.shouldUpgradeOs()) return false; return controller().serviceRegistry().configServer().nodeRepository() .targetVersionsOf(zone.getId()) .osVersion(application.nodeType()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java index 19f9542c679..7e4fc285793 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.slime.Cursor; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index df9491c5d22..ca080078328 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -43,7 +43,6 @@ import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ProtonMetrics; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; @@ -155,8 +154,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private static final ObjectMapper jsonMapper = new ObjectMapper(); - private static final String OPTIONAL_PREFIX = "/api"; - private final Controller controller; private final AccessControlRequests accessControlRequests; private final TestConfigSerializer testConfigSerializer; @@ -179,7 +176,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { @Override public HttpResponse handle(HttpRequest request) { try { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + Path path = new Path(request.getUri()); switch (request.getMethod()) { case GET: return handleGET(path, request); case PUT: return handlePUT(path, request); @@ -1841,99 +1838,30 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return ErrorResponse.badRequest("Missing required form part 'deployOptions'"); Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get(); - /* - * Special handling of the proxy application (the only system application with an application package) - * Setting any other deployOptions here is not supported for now (e.g. specifying version), but - * this might be handy later to handle emergency downgrades. - */ - boolean isZoneApplication = SystemApplication.proxy.id().equals(applicationId); - if (isZoneApplication) { // TODO jvenstad: Separate out. - // Make it explicit that version is not yet supported here - String versionStr = deployOptions.field("vespaVersion").asString(); - boolean versionPresent = !versionStr.isEmpty() && !versionStr.equals("null"); - if (versionPresent) { - throw new RuntimeException("Version not supported for system applications"); - } - // To avoid second guessing the orchestrated upgrades of system applications - // we don't allow to deploy these during an system upgrade (i.e when new vespa is being rolled out) - VersionStatus versionStatus = controller.readVersionStatus(); - if (versionStatus.isUpgrading()) { - throw new IllegalArgumentException("Deployment of system applications during a system upgrade is not allowed"); - } - Optional<VespaVersion> systemVersion = versionStatus.systemVersion(); - if (systemVersion.isEmpty()) { - throw new IllegalArgumentException("Deployment of system applications is not permitted until system version is determined"); - } - ActivateResult result = controller.applications() - .deploySystemApplicationPackage(SystemApplication.proxy, zone, systemVersion.get().versionNumber()); - return new SlimeJsonResponse(toSlime(result)); + // Resolve system application + Optional<SystemApplication> systemApplication = SystemApplication.matching(applicationId); + if (systemApplication.isEmpty() || !systemApplication.get().hasApplicationPackage()) { + return ErrorResponse.badRequest("Deployment of " + applicationId + " is not supported through this API"); } - /* - * Normal applications from here - */ - - Optional<ApplicationPackage> applicationPackage = Optional.ofNullable(dataParts.get("applicationZip")) - .map(ApplicationPackage::new); - Optional<Application> application = controller.applications().getApplication(TenantAndApplicationId.from(applicationId)); - - Inspector sourceRevision = deployOptions.field("sourceRevision"); - Inspector buildNumber = deployOptions.field("buildNumber"); - if (sourceRevision.valid() != buildNumber.valid()) - throw new IllegalArgumentException("Source revision and build number must both be provided, or not"); - - Optional<ApplicationVersion> applicationVersion = Optional.empty(); - if (sourceRevision.valid()) { - if (applicationPackage.isPresent()) - throw new IllegalArgumentException("Application version and application package can't both be provided."); - - applicationVersion = Optional.of(ApplicationVersion.from(toSourceRevision(sourceRevision), - buildNumber.asLong())); - applicationPackage = Optional.of(controller.applications().getApplicationPackage(applicationId, - applicationVersion.get())); + // Make it explicit that version is not yet supported here + String vespaVersion = deployOptions.field("vespaVersion").asString(); + if (!vespaVersion.isEmpty() && !vespaVersion.equals("null")) { + return ErrorResponse.badRequest("Specifying version for " + applicationId + " is not permitted"); } - boolean deployDirectly = deployOptions.field("deployDirectly").asBool(); - Optional<Version> vespaVersion = optional("vespaVersion", deployOptions).map(Version::new); - - if (deployDirectly && applicationPackage.isEmpty() && applicationVersion.isEmpty() && vespaVersion.isEmpty()) { - - // Redeploy the existing deployment with the same versions. - Optional<Deployment> deployment = controller.applications().getInstance(applicationId) - .map(Instance::deployments) - .flatMap(deployments -> Optional.ofNullable(deployments.get(zone))); - - if(deployment.isEmpty()) - throw new IllegalArgumentException("Can't redeploy application, no deployment currently exist"); - - ApplicationVersion version = deployment.get().applicationVersion(); - if(version.isUnknown()) - throw new IllegalArgumentException("Can't redeploy application, application version is unknown"); - - applicationVersion = Optional.of(version); - vespaVersion = Optional.of(deployment.get().version()); - applicationPackage = Optional.of(controller.applications().getApplicationPackage(applicationId, - applicationVersion.get())); + // To avoid second guessing the orchestrated upgrades of system applications + // we don't allow to deploy these during an system upgrade (i.e when new vespa is being rolled out) + VersionStatus versionStatus = controller.readVersionStatus(); + if (versionStatus.isUpgrading()) { + throw new IllegalArgumentException("Deployment of system applications during a system upgrade is not allowed"); } - - // TODO: get rid of the json object - DeployOptions deployOptionsJsonClass = new DeployOptions(deployDirectly, - vespaVersion, - deployOptions.field("ignoreValidationErrors").asBool(), - deployOptions.field("deployCurrentVersion").asBool()); - - applicationPackage.ifPresent(aPackage -> controller.applications().verifyApplicationIdentityConfiguration(applicationId.tenant(), - Optional.of(applicationId.instance()), - Optional.of(zone), - aPackage, - Optional.of(requireUserPrincipal(request)))); - - ActivateResult result = controller.applications().deploy(applicationId, - zone, - applicationPackage, - applicationVersion, - deployOptionsJsonClass); - + Optional<VespaVersion> systemVersion = versionStatus.systemVersion(); + if (systemVersion.isEmpty()) { + throw new IllegalArgumentException("Deployment of system applications is not permitted until system version is determined"); + } + ActivateResult result = controller.applications() + .deploySystemApplicationPackage(systemApplication.get(), zone, systemVersion.get().versionNumber()); return new SlimeJsonResponse(toSlime(result)); } @@ -2055,6 +1983,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { toSlime(tenantQuota, usedQuota, object.setObject("quota")); + cloudTenant.archiveAccessRole().ifPresent(role -> object.setString("archiveAccessRole", role)); + break; } default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java index d10a4879bf5..26c4bf6292a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiHandler.java @@ -37,7 +37,6 @@ import java.util.logging.Logger; public class AthenzApiHandler extends LoggingRequestHandler { private final static Logger log = Logger.getLogger(AthenzApiHandler.class.getName()); - private static final String OPTIONAL_PREFIX = "/api"; private final AthenzFacade athenz; private final AthenzDomain sandboxDomain; @@ -70,7 +69,7 @@ public class AthenzApiHandler extends LoggingRequestHandler { } private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + Path path = new Path(request.getUri()); if (path.matches("/athenz/v1")) return root(request); if (path.matches("/athenz/v1/domains")) return domainList(request); if (path.matches("/athenz/v1/properties")) return properties(); @@ -80,7 +79,7 @@ public class AthenzApiHandler extends LoggingRequestHandler { } private HttpResponse post(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + Path path = new Path(request.getUri()); if (path.matches("/athenz/v1/user")) return signup(request); return ErrorResponse.notFoundError(String.format("No '%s' handler at '%s'", request.getMethod(), request.getUri().getPath())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index 88191bc836b..c56c2e93f65 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.JacksonJsonResponse; import com.yahoo.restapi.MessageResponse; @@ -55,7 +54,6 @@ import java.util.stream.Collectors; */ public class BillingApiHandler extends LoggingRequestHandler { - private static final String OPTIONAL_PREFIX = "/api"; private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); private final BillingController billingController; @@ -73,7 +71,7 @@ public class BillingApiHandler extends LoggingRequestHandler { @Override public HttpResponse handle(HttpRequest request) { try { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + Path path = new Path(request.getUri()); String userId = userIdOrThrow(request); switch (request.getMethod()) { case GET: diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java index 09114a03522..b522469f179 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.changemanagement; import com.yahoo.config.provision.Environment; @@ -188,4 +189,4 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { return Optional.empty(); } -}
\ No newline at end of file +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java index f65f7534476..1bb3b1c5de8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java @@ -33,7 +33,6 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { private static final ZoneId CONTROLLER_ZONE = ZoneId.from("prod", "controller"); private static final URI CONTROLLER_URI = URI.create("https://localhost:4443/"); - private static final String OPTIONAL_PREFIX = "/api"; private static final List<String> WHITELISTED_APIS = List.of("/flags/v1/", "/nodes/v2/", "/orchestrator/v1/"); private final ZoneRegistry zoneRegistry; @@ -70,7 +69,7 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { } private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + Path path = new Path(request.getUri()); if (path.matches("/configserver/v1")) { return root(request); } @@ -78,7 +77,7 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { } private HttpResponse proxy(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + Path path = new Path(request.getUri()); if ( ! path.matches("/configserver/v1/{environment}/{region}/{*}")) { return ErrorResponse.notFoundError("Nothing at " + path); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index da13a84a3d4..552e22e9a2c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -49,8 +49,6 @@ import static java.util.stream.Collectors.toUnmodifiableMap; @SuppressWarnings("unused") // Injected public class DeploymentApiHandler extends LoggingRequestHandler { - private static final String OPTIONAL_PREFIX = "/api"; - private final Controller controller; public DeploymentApiHandler(LoggingRequestHandler.Context parentCtx, Controller controller) { @@ -77,7 +75,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { } private HttpResponse handleGET(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + Path path = new Path(request.getUri()); if (path.matches("/deployment/v1/")) return root(request); return ErrorResponse.notFoundError("Nothing at " + path); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index f26477beb3b..828e7e63483 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -42,7 +42,6 @@ import java.util.stream.Collectors; */ public class RoutingApiHandler extends AuditLoggingRequestHandler { - private static final String OPTIONAL_PREFIX = "/api"; private final Controller controller; public RoutingApiHandler(Context ctx, Controller controller) { @@ -53,7 +52,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { @Override public HttpResponse auditAndHandle(HttpRequest request) { try { - var path = new Path(request.getUri(), OPTIONAL_PREFIX); + var path = new Path(request.getUri()); switch (request.getMethod()) { case GET: return get(path, request); case POST: return post(path); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index e097b82b7d0..039e9b64df7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -64,7 +64,6 @@ import java.util.stream.Collectors; public class UserApiHandler extends LoggingRequestHandler { private final static Logger log = Logger.getLogger(UserApiHandler.class.getName()); - private static final String optionalPrefix = "/api"; private final UserManagement users; private final Controller controller; @@ -83,7 +82,7 @@ public class UserApiHandler extends LoggingRequestHandler { @Override public HttpResponse handle(HttpRequest request) { try { - Path path = new Path(request.getUri(), optionalPrefix); + Path path = new Path(request.getUri()); switch (request.getMethod()) { case GET: return handleGET(path, request); case POST: return handlePOST(path, request); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index abbbbef82c7..fd6dfa61180 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -29,8 +29,6 @@ import java.util.stream.Collectors; @SuppressWarnings("unused") public class ZoneApiHandler extends LoggingRequestHandler { - private static final String OPTIONAL_PREFIX = "/api"; - private final ZoneRegistry zoneRegistry; public ZoneApiHandler(LoggingRequestHandler.Context parentCtx, ServiceRegistry serviceRegistry) { @@ -57,7 +55,7 @@ public class ZoneApiHandler extends LoggingRequestHandler { } private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + Path path = new Path(request.getUri()); if (path.matches("/zone/v1")) { return root(request); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index b5126b7973b..4f9702669dd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -12,6 +12,7 @@ import java.time.Instant; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.regex.Pattern; /** * A paying tenant in a Vespa cloud service. @@ -20,6 +21,8 @@ import java.util.Optional; */ public class CloudTenant extends Tenant { + private static final Pattern VALID_ARCHIVE_ACCESS_ROLE_PATTERN = Pattern.compile("arn:aws:iam::\\d{12}:.+"); + private final Optional<Principal> creator; private final BiMap<PublicKey, Principal> developerKeys; private final TenantInfo info; @@ -36,6 +39,9 @@ public class CloudTenant extends Tenant { this.info = Objects.requireNonNull(info); this.tenantSecretStores = tenantSecretStores; this.archiveAccessRole = archiveAccessRole; + if (!archiveAccessRole.map(role -> VALID_ARCHIVE_ACCESS_ROLE_PATTERN.matcher(role).matches()).orElse(true)) + throw new IllegalArgumentException(String.format("Invalid archive access role '%s': Must match expected pattern: '%s'", + archiveAccessRole.get(), VALID_ARCHIVE_ACCESS_ROLE_PATTERN.pattern())); } /** Creates a tenant with the given name, provided it passes validation. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index ac9f32a2ae6..226852f1f3d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -63,7 +63,7 @@ public class OsVersionStatus { for (var application : SystemApplication.all()) { for (var zone : zonesToUpgrade(controller)) { - if (!application.shouldUpgradeOsIn()) continue; + if (!application.shouldUpgradeOs()) continue; var targetOsVersion = controller.serviceRegistry().configServer().nodeRepository() .targetVersionsOf(zone.getId()) .osVersion(application.nodeType()) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 07c0a6ca122..362c980a906 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -13,12 +13,10 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; @@ -32,7 +30,6 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -580,41 +577,6 @@ public class ControllerTest { } @Test - public void testIntegrationTestDeployment() { - Version six = Version.fromString("6.1"); - tester.controllerTester().zoneRegistry().setSystemName(SystemName.cd); - tester.controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("prod.cd-us-central-1")); - tester.configServer().bootstrap(List.of(ZoneId.from("prod.cd-us-central-1")), SystemApplication.all()); - tester.controllerTester().upgradeSystem(six); - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .majorVersion(6) - .region("cd-us-central-1") - .build(); - - // Create application - var context = tester.newDeploymentContext(); - - // Direct deploy is allowed when deployDirectly is true - ZoneId zone = ZoneId.from("prod", "cd-us-central-1"); - // Same options as used in our integration tests - DeployOptions options = new DeployOptions(true, Optional.empty(), false, - false); - tester.controller().applications().deploy(context.instanceId(), zone, Optional.of(applicationPackage), options); - - assertTrue("Application deployed and activated", - tester.configServer().application(context.instanceId(), zone).get().activated()); - - assertTrue("No job status added", - context.instanceJobs().isEmpty()); - - Version seven = Version.fromString("7.2"); - tester.controllerTester().upgradeSystem(seven); - tester.upgrader().maintain(); - tester.controller().applications().deploy(context.instanceId(), zone, Optional.of(applicationPackage), options); - assertEquals(six, context.instance().deployments().get(zone).version()); - } - - @Test public void testDevDeployment() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder().build(); @@ -625,7 +587,7 @@ public class ControllerTest { .setRoutingMethod(ZoneApiMock.from(zone), RoutingMethod.shared, RoutingMethod.sharedLayer4); // Deploy - tester.controller().applications().deploy(context.instanceId(), zone, Optional.of(applicationPackage), DeployOptions.none()); + context.runJob(zone, applicationPackage); assertTrue("Application deployed and activated", tester.configServer().application(context.instanceId(), zone).get().activated()); assertTrue("No job status added", @@ -682,10 +644,10 @@ public class ControllerTest { .region("us-west-1") .build(); - ZoneId zone = ZoneId.from("prod", "us-west-1"); - tester.controller().applications().deploy(context.instanceId(), zone, Optional.of(applicationPackage), DeployOptions.none()); - tester.controller().applications().deactivate(context.instanceId(), ZoneId.from(Environment.prod, RegionName.from("us-west-1"))); - tester.controller().applications().deactivate(context.instanceId(), ZoneId.from(Environment.prod, RegionName.from("us-west-1"))); + ZoneId zone = ZoneId.from(Environment.prod, RegionName.from("us-west-1")); + context.runJob(zone, applicationPackage); + tester.controller().applications().deactivate(context.instanceId(), zone); + tester.controller().applications().deactivate(context.instanceId(), zone); } @Test @@ -748,7 +710,7 @@ public class ControllerTest { var devZone = ZoneId.from("dev", "us-east-1"); // Deploy app2 in a zone with shared routing - tester.controller().applications().deploy(context2.instanceId(), devZone, Optional.of(applicationPackage), DeployOptions.none()); + context2.runJob(devZone, applicationPackage); assertTrue("Application deployed and activated", tester.configServer().application(context2.instanceId(), devZone).get().activated()); assertTrue("Provisions certificate also in zone with routing layer", certificate.apply(context2.instance()).isPresent()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index f432a1f41ce..f8645139244 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -16,7 +16,6 @@ import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.api.OktaIdentityToken; import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock; @@ -27,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; @@ -347,21 +345,6 @@ public final class ControllerTester { return application; } - public void deploy(ApplicationId id, ZoneId zone, ApplicationPackage applicationPackage, boolean deployCurrentVersion) { - deploy(id, zone, Optional.of(applicationPackage), deployCurrentVersion); - } - - public void deploy(ApplicationId id, ZoneId zone, Optional<ApplicationPackage> applicationPackage, boolean deployCurrentVersion) { - deploy(id, zone, applicationPackage, deployCurrentVersion, Optional.empty()); - } - - public void deploy(ApplicationId id, ZoneId zone, Optional<ApplicationPackage> applicationPackage, boolean deployCurrentVersion, Optional<Version> version) { - controller().applications().deploy(id, - zone, - applicationPackage, - new DeployOptions(false, version, false, deployCurrentVersion)); - } - private static Controller createController(CuratorDb curator, RotationsConfig rotationsConfig, AthenzDbMock athensDb, ServiceRegistryMock serviceRegistry) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java index f626a597d4a..1849be9b6bd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.application.api.DeploymentSpec; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java index 737807b5dd3..91d71a1aa25 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java index fa0af070e1e..e30f044c579 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.certificate; import com.yahoo.config.application.api.DeploymentSpec; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 289d48d3241..4a4159180b5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -328,11 +328,16 @@ public class DeploymentContext { return runJob(type); } - /** Runs a deployment of the given package to the given dev/perf job. */ + /** Runs a deployment of the given package to the given manually deployable job. */ public DeploymentContext runJob(JobType type, ApplicationPackage applicationPackage) { return runJob(type, applicationPackage, null); } + /** Runs a deployment of the given package to the given manually deployable zone. */ + public DeploymentContext runJob(ZoneId zone, ApplicationPackage applicationPackage) { + return runJob(JobType.from(tester.controller().system(), zone).get(), applicationPackage, null); + } + /** Pulls the ready job trigger, and then runs the whole of the given job, successfully. */ public DeploymentContext runJob(JobType type) { var job = jobId(type); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 724ba61da2b..81c9f51278e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -628,13 +627,8 @@ public class DeploymentTriggerTest { assertEquals("First deployment gets system version", version1, app1.application().oldestDeployedPlatform().get()); assertEquals(version1, tester.configServer().lastPrepareVersion().get()); - // Unexpected deployment is ignored - Version version2 = new Version(version1.getMajor(), version1.getMinor() + 1); - tester.applications().deploy(app1.instanceId(), ZoneId.from("prod", "us-west-1"), - Optional.empty(), new DeployOptions(false, Optional.of(version2), false, false)); - assertEquals(version1, app1.deployment(ZoneId.from("prod", "us-west-1")).version()); - // Application change after a new system version, and a region added + Version version2 = new Version(version1.getMajor(), version1.getMinor() + 1); tester.controllerTester().upgradeSystem(version2); applicationPackage = new ApplicationPackageBuilder() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java index 5ec14ee6156..a4fb4669237 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; @@ -124,4 +125,4 @@ public class ChangeManagementAssessorTest { return node; } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java index b4df16348d2..1ce59587d6c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.ControllerTester; @@ -56,4 +57,4 @@ public class ChangeRequestMaintainerTest { .build(); } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java index 4f12861870a..d14d4014b48 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java index 00f5335bd82..ff59e14947b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index 9790c8244c8..e123c4cca62 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -120,7 +120,7 @@ public class TenantSerializerTest { new TenantSecretStore("ss1", "123", "role1"), new TenantSecretStore("ss2", "124", "role2") ), - Optional.of("role3") + Optional.of("arn:aws:iam::123456789012:role/my-role") ); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.info(), serialized.info()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index 2bf6eb39089..10f143a8e96 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -142,12 +142,16 @@ public class ContainerTester { expectedStatusCode); } - public void assertResponse(Supplier<Request> requestSupplier, Consumer<Response> responseAssertion, int expectedStatusCode) { + public void assertResponse(Supplier<Request> requestSupplier, ConsumerThrowingException<Response> responseAssertion, int expectedStatusCode) { var request = requestSupplier.get(); FilterResult filterResult = invokeSecurityFilters(request); request = filterResult.request; Response response = filterResult.response != null ? filterResult.response : container.handleRequest(request); - responseAssertion.accept(response); + try { + responseAssertion.accept(response); + } catch (Exception e) { + throw new RuntimeException(e); + } assertEquals("Status code", expectedStatusCode, response.getStatus()); } @@ -203,5 +207,9 @@ public class ContainerTester { } } + @FunctionalInterface + public interface ConsumerThrowingException<T> { + void accept(T t) throws Exception; + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java index b935f8cbbe4..538e98c4c83 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -36,14 +36,11 @@ public class ControllerContainerCloudTest extends ControllerContainerTest { " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + " <binding>http://*/application/v4/*</binding>\n" + - " <binding>http://*/api/application/v4/*</binding>\n" + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'>\n" + " <binding>http://*/zone/v1</binding>\n" + " <binding>http://*/zone/v1/*</binding>\n" + - " <binding>http://*/api/zone/v1</binding>\n" + - " <binding>http://*/api/zone/v1/*</binding>\n" + " </handler>\n" + " <http>\n" + @@ -88,6 +85,7 @@ public class ControllerContainerCloudTest extends ControllerContainerTest { public RequestBuilder principal(String principal) { this.principal = new SimplePrincipal(principal); return this; } public RequestBuilder user(User user) { this.user = user; return this; } public RequestBuilder roles(Set<Role> roles) { this.roles = roles; return this; } + public RequestBuilder roles(Role... roles) { return roles(Set.of(roles)); } @Override public Request get() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 50123e497dc..e92c229d4bc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -75,7 +75,6 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockUserManagement'/>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.DeploymentApiHandler'>\n" + " <binding>http://*/deployment/v1/*</binding>\n" + - " <binding>http://*/api/deployment/v1/*</binding>\n" + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.BadgeApiHandler'>\n" + " <binding>http://*/badge/v1/*</binding>\n" + @@ -93,8 +92,6 @@ public class ControllerContainerTest { " <handler id='com.yahoo.vespa.hosted.controller.restapi.configserver.ConfigServerApiHandler'>\n" + " <binding>http://*/configserver/v1</binding>\n" + " <binding>http://*/configserver/v1/*</binding>\n" + - " <binding>http://*/api/configserver/v1</binding>\n" + - " <binding>http://*/api/configserver/v1/*</binding>\n" + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.flags.AuditedFlagsHandler'>\n" + " <binding>http://*/flags/v1</binding>\n" + @@ -102,11 +99,9 @@ public class ControllerContainerTest { " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.user.UserApiHandler'>\n" + " <binding>http://*/user/v1/*</binding>\n" + - " <binding>http://*/api/user/v1/*</binding>\n" + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.routing.RoutingApiHandler'>\n" + " <binding>http://*/routing/v1/*</binding>\n" + - " <binding>http://*/api/routing/v1/*</binding>\n" + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.changemanagement.ChangeManagementApiHandler'>\n" + " <binding>http://*/changemanagement/v1/*</binding>\n" + @@ -128,7 +123,6 @@ public class ControllerContainerTest { " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.athenz.AthenzApiHandler'>\n" + " <binding>http://*/athenz/v1/*</binding>\n" + - " <binding>http://*/api/athenz/v1/*</binding>\n" + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'>\n" + " <binding>http://*/zone/v1</binding>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 62865ea9be2..4fb91639daa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -4,13 +4,12 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.LockedTenant; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -29,9 +28,14 @@ import java.util.Collections; import java.util.Optional; import java.util.Set; -import static com.yahoo.application.container.handler.Request.Method.*; +import static com.yahoo.application.container.handler.Request.Method.DELETE; +import static com.yahoo.application.container.handler.Request.Method.GET; +import static com.yahoo.application.container.handler.Request.Method.POST; +import static com.yahoo.application.container.handler.Request.Method.PUT; import static com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiTest.createApplicationSubmissionData; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -153,7 +157,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { public void validate_secret_store() { deployApplication(); var secretStoreRequest = - request("/application/v4/tenant/scoober/secret-store/secret-foo/validate?aws-region=us-west-1¶meter-name=foo&application-id=scoober.albums.default&zone=prod.us-central-1", GET) + request("/application/v4/tenant/scoober/secret-store/secret-foo/validate?aws-region=us-west-1¶meter-name=foo&application-id=scoober.albums.default&zone=prod.aws-us-east-1c", GET) .roles(Set.of(Role.developer(tenantName))); tester.assertResponse(secretStoreRequest, "{" + "\"error-code\":\"NOT_FOUND\"," + @@ -167,9 +171,9 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { // ConfigServerMock returns message on format deployment.toString() + " - " + tenantSecretStore.toString() secretStoreRequest = - request("/application/v4/tenant/scoober/secret-store/secret-foo/validate?aws-region=us-west-1¶meter-name=foo&application-id=scoober.albums.default&zone=prod.us-central-1", GET) + request("/application/v4/tenant/scoober/secret-store/secret-foo/validate?aws-region=us-west-1¶meter-name=foo&application-id=scoober.albums.default&zone=prod.aws-us-east-1c", GET) .roles(Set.of(Role.developer(tenantName))); - tester.assertResponse(secretStoreRequest, "{\"target\":\"scoober.albums in prod.us-central-1\",\"result\":{\"settings\":{\"name\":\"foo\",\"role\":\"vespa-secretstore-access\",\"awsId\":\"892075328880\",\"externalId\":\"*****\",\"region\":\"us-east-1\"},\"status\":\"ok\"}}", 200); + tester.assertResponse(secretStoreRequest, "{\"target\":\"scoober.albums in prod.aws-us-east-1c\",\"result\":{\"settings\":{\"name\":\"foo\",\"role\":\"vespa-secretstore-access\",\"awsId\":\"892075328880\",\"externalId\":\"*****\",\"region\":\"us-east-1\"},\"status\":\"ok\"}}", 200); } @Test @@ -193,6 +197,29 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { assertEquals(0, tenant.tenantSecretStores().size()); } + @Test + public void archive_uri_test() { + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertFalse(response.getBodyAsString().contains("archiveAccessRole")), + 200); + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access", PUT) + .data("{\"role\":\"dummy\"}").roles(Role.administrator(tenantName)), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid archive access role 'dummy': Must match expected pattern: 'arn:aws:iam::\\\\d{12}:.+'\"}", 400); + + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access", PUT) + .data("{\"role\":\"arn:aws:iam::123456789012:role/my-role\"}").roles(Role.administrator(tenantName)), + "{\"message\":\"Archive access role set to 'arn:aws:iam::123456789012:role/my-role' for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertTrue(response.getBodyAsString().contains("\"archiveAccessRole\":\"arn:aws:iam::123456789012:role/my-role\"")), + 200); + + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access", DELETE).roles(Role.administrator(tenantName)), + "{\"message\":\"Archive access role removed for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertFalse(response.getBodyAsString().contains("archiveAccessRole")), + 200); + } + private ApplicationPackageBuilder prodBuilder() { return new ApplicationPackageBuilder() .instances("default") @@ -219,12 +246,14 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { var applicationPackage = new ApplicationPackageBuilder() .instances("default") .globalServiceId("foo") - .region("us-central-1") + .region("aws-us-east-1c") .build(); + tester.controller().jobController().deploy(ApplicationId.from("scoober", "albums", "default"), + JobType.productionAwsUsEast1c, + Optional.empty(), + applicationPackage); + - tester.controller().applications().deploy(ApplicationId.from("scoober", "albums", "default"), - ZoneId.from("prod", "us-central-1"), - Optional.of(applicationPackage), - new DeployOptions(true, Optional.empty(), false, false)); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 673563097f4..c69d2069650 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -28,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.RoutingController; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ProtonMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; @@ -54,7 +53,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; @@ -303,32 +301,6 @@ public class ApplicationApiTest extends ControllerContainerTest { app1.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsCentral1); - // POST an application deployment to a production zone - operator emergency deployment - fails since package is unknown - entity = createApplicationDeployData(Optional.empty(), - Optional.of(ApplicationVersion.from(DeploymentContext.defaultSourceRevision, 666)), - true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/", POST) - .data(entity) - .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No application package found for tenant1.application1 with version 1.0.666-commit1\"}", - 400); - - // POST an application deployment to a production zone - operator emergency deployment - works with known package - entity = createApplicationDeployData(Optional.empty(), - Optional.of(ApplicationVersion.from(DeploymentContext.defaultSourceRevision, 1)), - true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/", POST) - .data(entity) - .userIdentity(HOSTED_VESPA_OPERATOR), - new File("deploy-result.json")); - - // POST an application deployment to a production zone - operator emergency deployment - chooses latest package without arguments - entity = createApplicationDeployData(Optional.empty(), true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/", POST) - .data(entity) - .userIdentity(HOSTED_VESPA_OPERATOR), - new File("deploy-result.json")); - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .instances("instance1") .globalServiceId("foo") @@ -702,14 +674,14 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Deactivated tenant1.application1.instance1 in prod.us-central-1\"}"); // Setup for test config tests - tester.controller().applications().deploy(ApplicationId.from("tenant1", "application1", "default"), - ZoneId.from("prod", "us-central-1"), - Optional.of(applicationPackageDefault), - new DeployOptions(true, Optional.empty(), false, false)); - tester.controller().applications().deploy(ApplicationId.from("tenant1", "application1", "my-user"), - ZoneId.from("dev", "us-east-1"), - Optional.of(applicationPackageDefault), - new DeployOptions(false, Optional.empty(), false, false)); + tester.controller().jobController().deploy(ApplicationId.from("tenant1", "application1", "default"), + JobType.productionUsCentral1, + Optional.empty(), + applicationPackageDefault); + tester.controller().jobController().deploy(ApplicationId.from("tenant1", "application1", "my-user"), + JobType.devUsEast1, + Optional.empty(), + applicationPackageDefault); // GET test-config for local tests against a dev deployment tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/my-user/job/dev-us-east-1/test-config", GET) @@ -989,13 +961,6 @@ public class ApplicationApiTest extends ControllerContainerTest { // Add build service to operator role addUserToHostedOperatorRole(HostedAthenzIdentities.from(SCREWDRIVER_ID)); - // POST (deploy) an application to a prod zone - allowed when project ID is not specified - MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/deploy", POST) - .data(entity) - .screwdriverIdentity(SCREWDRIVER_ID), - new File("deploy-result.json")); - // POST (deploy) a system application with an application package MultiPartStreamer noAppEntity = createApplicationDeployData(Optional.empty(), true); tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/routing/environment/prod/region/us-central-1/instance/default/deploy", POST) @@ -1008,12 +973,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(noAppEntity) .userIdentity(HOSTED_VESPA_OPERATOR), new File("deploy-result.json")); - - // POST (deploy) a system application without an application package - tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/proxy-host/environment/prod/region/us-central-1/instance/instance1/deploy", POST) - .data(noAppEntity) - .userIdentity(HOSTED_VESPA_OPERATOR), - new File("deploy-no-deployment.json"), 400); } @Test @@ -1178,33 +1137,12 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid build number: For input string: \\\"foobar\\\"\"}", 400); - // POST (deploy) an application with an invalid application package + // POST (deploy) an application to legacy deploy path MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), - new File("deploy-failure.json"), 400); - - // POST (deploy) an application without available capacity - configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", "Out of capacity", ConfigServerException.ErrorCode.OUT_OF_CAPACITY, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/instance1/deploy", POST) - .data(entity) - .userIdentity(USER_ID), - new File("deploy-out-of-capacity.json"), 400); - - // POST (deploy) an application where activation fails - configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to activate application", "Activation conflict", ConfigServerException.ErrorCode.ACTIVATION_CONFLICT, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/instance1/deploy", POST) - .data(entity) - .userIdentity(USER_ID), - new File("deploy-activation-conflict.json"), 409); - - // POST (deploy) an application where we get an internal server error - configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to deploy application", "Internal server error", ConfigServerException.ErrorCode.INTERNAL_SERVER_ERROR, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/instance1/deploy", POST) - .data(entity) - .userIdentity(USER_ID), - new File("deploy-internal-server-error.json"), 500); + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Deployment of tenant1.application1.instance1 is not supported through this API\"}", 400); // DELETE tenant which has an application tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE) @@ -1248,7 +1186,7 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Tenant 'my-tenant' already exists\"}", 400); } - + @Test public void testAuthorization() { UserId authorizedUser = USER_ID; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index c43abf276c5..f574d6bc3f1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -5,7 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.test.json.JsonTestHelper; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -171,10 +170,7 @@ public class JobControllerApiHandlerHelperTest { var region = "us-west-1"; var applicationPackage = new ApplicationPackageBuilder().region(region).build(); // Deploy directly to production zone, like integration tests. - tester.controller().applications().deploy(tester.instance().id(), ZoneId.from("prod", region), - Optional.of(applicationPackage), - new DeployOptions(true, Optional.empty(), - false, false)); + tester.controller().jobController().deploy(tester.instance().id(), productionUsWest1, Optional.empty(), applicationPackage); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root/")), "jobs-direct-deployment.json"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/cost-report.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/cost-report.json deleted file mode 100644 index 8f6dbf17d51..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/cost-report.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "month": "2019-09", - "items": [ - { - "applicationId":"tenant1:application1:instance1", - "zoneId":"prod.us-south-1", - "cpu": {"usage":7.0,"charge":35}, - "memory": {"usage":600.0,"charge":23}, - "disk": {"usage":1000.0,"charge":10} - }, - { - "applicationId":"tenant1:application1:instance1", - "zoneId":"prod.us-north-1", - "cpu": {"usage":2.0,"charge":10}, - "memory": {"usage":3.0,"charge":20}, - "disk": {"usage":4.0,"charge":30} - } - ] - -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/create-user-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/create-user-response.json deleted file mode 100644 index a6130122650..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/create-user-response.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "message":"Created user 'by-new-user'" -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-activation-conflict.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-activation-conflict.json deleted file mode 100644 index 39d4faa53c9..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-activation-conflict.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "error-code":"ACTIVATION_CONFLICT", - "message":"Failed to activate application: Activation conflict" -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-failure.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-failure.json deleted file mode 100644 index c8802cce57b..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-failure.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "error-code":"INVALID_APPLICATION_PACKAGE", - "message":"Failed to prepare application: Invalid application package" -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-internal-server-error.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-internal-server-error.json deleted file mode 100644 index 9a845e2a7d6..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-internal-server-error.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "error-code":"INTERNAL_SERVER_ERROR", - "message":"Failed to deploy application: Internal server error" -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-no-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-no-deployment.json deleted file mode 100644 index f90420f28d7..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-no-deployment.json +++ /dev/null @@ -1 +0,0 @@ -{"error-code":"BAD_REQUEST","message":"Can't redeploy application, no deployment currently exist"}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-out-of-capacity.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-out-of-capacity.json deleted file mode 100644 index 0bdf5a2653c..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-out-of-capacity.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "error-code":"OUT_OF_CAPACITY", - "message":"Failed to prepare application: Out of capacity" -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 961d36bd2f3..55be0881ec2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -418,6 +418,7 @@ "targetPlatform": "6.1.0", "targetApplication": { "build": 1, + "compileVersion": "6.1.0", "sourceUrl": "repository1/tree/commit1", "commit": "commit1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list-with-user.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list-with-user.json deleted file mode 100644 index 774b80f8b0c..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list-with-user.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "tenant": "by-myuser", - "metaData": { - "type": "USER" - }, - "url": "http://localhost:8080/application/v4/tenant/by-myuser" - }, - { - "tenant": "tenant1", - "metaData": { - "type": "ATHENS", - "athensDomain": "domain1", - "property": "property1" - }, - "url": "http://localhost:8080/application/v4/tenant/tenant1" - } -] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list.json deleted file mode 100644 index d7ec9a738f2..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "tenant": "tenant1", - "metaData": { - "type": "ATHENS", - "athensDomain": "domain1", - "property": "property1" - }, - "url": "http://localhost:8080/application/v4/tenant/tenant1" - } -] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java index 8d3f2d584b8..bf67e06db21 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/athenz/AthenzApiTest.java @@ -49,7 +49,7 @@ public class AthenzApiTest extends ControllerContainerTest { new File("property-list.json")); // POST user signup - tester.assertResponse(authenticatedRequest("http://localhost:8080/api/athenz/v1/user", "", Request.Method.POST), + tester.assertResponse(authenticatedRequest("http://localhost:8080/athenz/v1/user", "", Request.Method.POST), "{\"message\":\"User 'bob' added to admin role of 'vespa.vespa.tenants.sandbox'\"}"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java index 5b46df1ad1f..b88715efcc4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.config.provision.SystemName; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java index 28f2b836365..6f5aa776fef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java @@ -1,4 +1,4 @@ -// Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.changemanagement; import com.yahoo.application.container.handler.Request; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java index c414a3680fc..519c3f56976 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java @@ -66,7 +66,7 @@ public class ConfigServerApiHandlerTest extends ControllerContainerTest { assertLastRequest("https://cfg.prod.us-north-1.test.vip:4443/", "PUT"); // DELETE /configserver/v1/prod/us-north-1/nodes/v2/node/node1 - tester.assertResponse(operatorRequest("http://localhost:8080/api/configserver/v1/prod/controller/nodes/v2/node/node1", + tester.assertResponse(operatorRequest("http://localhost:8080/configserver/v1/prod/controller/nodes/v2/node/node1", "", Request.Method.DELETE), "ok"); assertLastRequest("https://localhost:4443/", "DELETE"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index f3c24458e6e..3d07b5d51d4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -74,7 +74,6 @@ public class DeploymentApiTest extends ControllerContainerTest { tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()))); tester.assertResponse(operatorRequest("http://localhost:8080/deployment/v1/"), new File("root.json")); - tester.assertResponse(operatorRequest("http://localhost:8080/api/deployment/v1/"), new File("root.json")); } private VersionStatus censorConfigServers(VersionStatus versionStatus) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java index df402e8c594..5bf7b03295f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; import com.yahoo.application.container.handler.Request; @@ -56,4 +57,4 @@ public class LastLoginUpdateFilterTest { request.getAttributes().put(SecurityContext.ATTRIBUTE_NAME, context); filter.filter(new ApplicationRequestToDiscFilterRequestWrapper(request)); } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index 0a9e2dac49e..b1bd7df059c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -77,12 +77,6 @@ public class RoutingApiTest extends ControllerContainerTest { tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/", "", Request.Method.GET), new File("discovery/environment.json")); - - // GET instance with api prefix (test that the /api prefix works) - tester.assertResponse(authenticatedRequest("http://localhost:8080/api/routing/v1/status/tenant/t1/application/a1/instance/default/", - "", - Request.Method.GET), - new File("discovery/instance_api.json")); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance_api.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance_api.json deleted file mode 100644 index a9e789d9fe9..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance_api.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "resources": [ - { - "url": "http://localhost:8080/api/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-east-3/" - }, - { - "url": "http://localhost:8080/api/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-west-1/" - } - ] -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java index 3a63caf52cd..acd481030e2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiOnPremTest.java @@ -62,7 +62,7 @@ public class UserApiOnPremTest extends ControllerContainerTest { } private Request createUserRequest(User user, AthenzIdentity identity) { - Request request = new Request("http://localhost:8080/api/user/v1/user"); + Request request = new Request("http://localhost:8080/user/v1/user"); Map<String, String> userAttributes = new HashMap<>(); userAttributes.put("email", user.email()); if (user.name() != null) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index dc765e01d4e..1ad705be0b7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -57,11 +57,6 @@ public class UserApiTest extends ControllerContainerCloudTest { .roles(operator), "[]"); - // GET at application/v4/tenant is available also under the /api prefix. - tester.assertResponse(request("/api/application/v4/tenant") - .roles(operator), - "[]"); - // POST a tenant is not available to everyone. tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) .data("{\"token\":\"hello\"}"), @@ -129,11 +124,6 @@ public class UserApiTest extends ControllerContainerCloudTest { .roles(Set.of(Role.administrator(id.tenant()))), new File("application-roles.json")); - // GET application role information is available also under the /api prefix. - tester.assertResponse(request("/api/user/v1/tenant/my-tenant/application/my-app") - .roles(Set.of(Role.administrator(id.tenant()))), - new File("application-roles.json")); - // POST a pem deploy key tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app/key", POST) .roles(Set.of(Role.developer(id.tenant()))) @@ -223,7 +213,7 @@ public class UserApiTest extends ControllerContainerCloudTest { Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter(), Role.hostedAccountant()); User user = new User("dev@domail", "Joe Developer", "dev", null); - tester.assertResponse(request("/api/user/v1/user") + tester.assertResponse(request("/user/v1/user") .roles(operator) .user(user), new File("user-without-applications.json")); @@ -246,13 +236,13 @@ public class UserApiTest extends ControllerContainerCloudTest { controller.createApplication("sandbox", "app2", "dev"); // Should still be empty because none of the roles explicitly refer to any of the applications - tester.assertResponse(request("/api/user/v1/user") + tester.assertResponse(request("/user/v1/user") .roles(operator) .user(user), new File("user-without-applications.json")); // Empty applications because tenant dummy does not exist - tester.assertResponse(request("/api/user/v1/user") + tester.assertResponse(request("/user/v1/user") .roles(Set.of(Role.administrator(TenantName.from("tenant1")), Role.developer(TenantName.from("tenant2")), Role.developer(TenantName.from("sandbox")), @@ -274,7 +264,7 @@ public class UserApiTest extends ControllerContainerCloudTest { controller.createTenant("tenant1", Tenant.Type.cloud); tester.assertResponse( - request("/api/user/v1/user").user(user), + request("/user/v1/user").user(user), new File("user-without-trial-capacity-cloud.json")); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java index d5031267b27..a8845fa92a3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java @@ -52,7 +52,7 @@ public class ZoneApiTest extends ControllerContainerCloudTest { new File("prod.json")); // GET /zone/v1/environment/dev/default - tester.assertResponse(request("/api/zone/v1/environment/dev/default") + tester.assertResponse(request("/zone/v1/environment/dev/default") .roles(everyone), new File("default-for-region.json")); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index fab61eeaec3..38f5a60cf8a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -19,7 +19,6 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.RoutingController; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -354,7 +353,7 @@ public class RoutingPoliciesTest { .exclusiveRoutingIn(zoneApi); // Deploy to dev - tester.controllerTester().controller().applications().deploy(context.instanceId(), zone, Optional.of(emptyApplicationPackage), DeployOptions.none()); + context.runJob(zone, emptyApplicationPackage); assertEquals("DeploymentSpec is not persisted", DeploymentSpec.empty, context.application().deploymentSpec()); context.flushDnsUpdates(); @@ -378,13 +377,14 @@ public class RoutingPoliciesTest { assertEquals(prodRecords, tester.recordNames()); // Deploy to dev under different instance - var devInstance = context.application().id().instance("user"); - tester.controllerTester().controller().applications().deploy(devInstance, zone, Optional.of(applicationPackage), DeployOptions.none()); + var devContext = tester.newDeploymentContext(context.application().id().instance("user")); + devContext.runJob(zone, applicationPackage); + assertEquals("DeploymentSpec is persisted", applicationPackage.deploymentSpec(), context.application().deploymentSpec()); context.flushDnsUpdates(); // Routing policy is created and DNS is updated - assertEquals(1, tester.policiesOf(devInstance).size()); + assertEquals(1, tester.policiesOf(devContext.instanceId()).size()); assertEquals(Sets.union(prodRecords, Set.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames()); } @@ -732,6 +732,10 @@ public class RoutingPoliciesTest { return tester.newDeploymentContext(tenant, application, instance); } + public DeploymentContext newDeploymentContext(ApplicationId instance) { + return tester.newDeploymentContext(instance); + } + public ControllerTester controllerTester() { return tester.controllerTester(); } diff --git a/document/src/vespa/document/fieldvalue/structuredcache.h b/document/src/vespa/document/fieldvalue/structuredcache.h index 24d1a3dbd78..3d4911477fd 100644 --- a/document/src/vespa/document/fieldvalue/structuredcache.h +++ b/document/src/vespa/document/fieldvalue/structuredcache.h @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once #include "fieldvalue.h" diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalVisitorSession.java index 06452ac1bba..1101ef0d08e 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalVisitorSession.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.local; import com.yahoo.document.Document; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java index c464bed5b87..bb4ed1367ae 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.jrt.slobrok.api.Mirror; diff --git a/fastos/src/vespa/fastos/file_rw_ops.cpp b/fastos/src/vespa/fastos/file_rw_ops.cpp index f5d81eab1cf..7285178c19b 100644 --- a/fastos/src/vespa/fastos/file_rw_ops.cpp +++ b/fastos/src/vespa/fastos/file_rw_ops.cpp @@ -1,4 +1,4 @@ -// Copyright 2020 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "file_rw_ops.h" #include <unistd.h> diff --git a/fastos/src/vespa/fastos/file_rw_ops.h b/fastos/src/vespa/fastos/file_rw_ops.h index 9328bdbf9b4..169b60eed45 100644 --- a/fastos/src/vespa/fastos/file_rw_ops.h +++ b/fastos/src/vespa/fastos/file_rw_ops.h @@ -1,4 +1,4 @@ -// Copyright 2020 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once diff --git a/flags/src/test/java/com/yahoo/vespa/flags/PermanentFlagsTest.java b/flags/src/test/java/com/yahoo/vespa/flags/PermanentFlagsTest.java index 3f43682cfb9..956048f6e24 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/PermanentFlagsTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/PermanentFlagsTest.java @@ -1,4 +1,5 @@ -package com.yahoo.vespa.flags;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.flags; import com.yahoo.vespa.flags.custom.HostResources; import com.yahoo.vespa.flags.custom.SharedHost; @@ -22,4 +23,4 @@ class PermanentFlagsTest { testGeneric(PermanentFlags.SHARED_HOST, sharedHost); } -}
\ No newline at end of file +} diff --git a/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java b/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java index b1d78fc09eb..258f29f4ced 100644 --- a/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java +++ b/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java @@ -1,4 +1,5 @@ -package ai.vespa.util.http.retry;// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.util.http.retry; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; @@ -127,4 +128,4 @@ public class DelayedResponseLevelRetryHandlerTest { return new BasicHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, statusCode, "reason phrase")); } -}
\ No newline at end of file +} diff --git a/jdisc-cloud-aws/pom.xml b/jdisc-cloud-aws/pom.xml index bbb31d380ff..1406a70ece0 100644 --- a/jdisc-cloud-aws/pom.xml +++ b/jdisc-cloud-aws/pom.xml @@ -1,7 +1,5 @@ <?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - --> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/TraceRenderer.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/TraceRenderer.java index 295786aa15d..41b88e08c19 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/TraceRenderer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/TraceRenderer.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.logging; import com.yahoo.data.access.Inspectable; diff --git a/jdisc_http_service/src/main/resources/configdefinitions/container.logging.connection-log.def b/jdisc_http_service/src/main/resources/configdefinitions/container.logging.connection-log.def index e845a8351d5..39841dba6f2 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/container.logging.connection-log.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/container.logging.connection-log.def @@ -1,3 +1,4 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=container.logging # Name of the cluster @@ -7,4 +8,4 @@ cluster string logDirectoryName string default="qrs" # Max queue length of file handler -queueSize int default=10000
\ No newline at end of file +queueSize int default=10000 diff --git a/jdisc_http_service/src/test/java/com/yahoo/container/logging/JsonConnectionLogWriterTest.java b/jdisc_http_service/src/test/java/com/yahoo/container/logging/JsonConnectionLogWriterTest.java index 15118b23f85..33ecb664af5 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/container/logging/JsonConnectionLogWriterTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/container/logging/JsonConnectionLogWriterTest.java @@ -1,5 +1,5 @@ -package com.yahoo.container.logging;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.logging; import com.yahoo.test.json.JsonTestHelper; import org.junit.jupiter.api.Test; @@ -41,4 +41,4 @@ class JsonConnectionLogWriterTest { String actualJson = out.toString(StandardCharsets.UTF_8); JsonTestHelper.assertJsonEquals(actualJson, expectedJson); } -}
\ No newline at end of file +} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessor.java index 292c6da3de2..7f58379fcb5 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessor.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessor.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.MetricsPacket; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java index 4d1d57644b5..6bbc71bb2cb 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessor.java index 45f211a5704..b4105ad32d4 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessor.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessor.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.MetricsPacket; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java index 8d525310d6f..f6bda2f9ef5 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.dimensions; import java.util.List; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/processing/MetricsProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/processing/MetricsProcessor.java index 7860825e075..7c4320efa75 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/processing/MetricsProcessor.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/processing/MetricsProcessor.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.metric.model.processing; import ai.vespa.metricsproxy.metric.model.MetricsPacket; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessorTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessorTest.java index 0f4c6a885ec..1a4168b74b1 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessorTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessorTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.DimensionId; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessorTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessorTest.java index c6f47d70a1f..c29fa3a872a 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessorTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessorTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.MetricsPacket; diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessorTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessorTest.java index 3237abdbfa3..43eacd766dc 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessorTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessorTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.DimensionId; diff --git a/model-integration/src/main/python/vespa-convert-tf2onnx.py b/model-integration/src/main/python/vespa-convert-tf2onnx.py index 1862072aebc..b349e7e9683 100755 --- a/model-integration/src/main/python/vespa-convert-tf2onnx.py +++ b/model-integration/src/main/python/vespa-convert-tf2onnx.py @@ -1,5 +1,4 @@ #! /usr/bin/env python3 - # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import onnx diff --git a/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py b/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py index ac00437d192..740059824b1 100755 --- a/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py +++ b/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py @@ -1,4 +1,5 @@ #! /usr/bin/env python3 +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # coding: utf-8 import json diff --git a/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py b/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py index 3e74e38da35..e4a634236f1 100755 --- a/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py +++ b/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py @@ -1,4 +1,5 @@ #! /usr/bin/env python3 +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # coding: utf-8 import json diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java index d57c680e190..c29c72657e5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentTask.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; import java.util.Arrays; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java index be0df437b7f..9341d45aa9a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.sync; import org.junit.Test; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java index bcf3651fdfe..898ddf57428 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.file; import org.junit.Test; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 14e68bc2f0f..a06ad89e299 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -2,9 +2,11 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import java.util.Optional; @@ -39,6 +41,7 @@ public class AllocationOptimizer { */ public Optional<AllocatableClusterResources> findBestAllocation(ResourceTarget target, AllocatableClusterResources current, + ClusterModel clusterModel, Limits limits) { int minimumNodes = AllocationOptimizer.minimumNodes; if (limits.isEmpty()) @@ -63,7 +66,7 @@ public class AllocationOptimizer { groups, nodeResourcesWith(nodesAdjustedForRedundancy, groupsAdjustedForRedundancy, - limits, current, target)); + limits, target, current, clusterModel)); var allocatableResources = AllocatableClusterResources.from(next, current.clusterSpec(), limits, hosts, nodeRepository); if (allocatableResources.isEmpty()) continue; @@ -82,20 +85,22 @@ public class AllocationOptimizer { private NodeResources nodeResourcesWith(int nodes, int groups, Limits limits, + ResourceTarget target, AllocatableClusterResources current, - ResourceTarget target) { - // Cpu: Scales with cluster size (TODO: Only reads, writes scales with group size) - // Memory and disk: Scales with group size + ClusterModel clusterModel) { double cpu, memory, disk; - int groupSize = nodes / groups; - if (current.clusterSpec().isStateful()) { // load scales with node share of content + if (current.clusterSpec().type() == ClusterSpec.Type.content) { // load scales with node share of content + // Cpu: Query cpu scales with cluster size, write cpu scales with group size + // Memory and disk: Scales with group size + // The fixed cost portion of cpu does not scale with changes to the node count - // TODO: Only for the portion of cpu consumed by queries - double cpuPerGroup = fixedCpuCostFraction * target.nodeCpu() + - (1 - fixedCpuCostFraction) * target.nodeCpu() * current.groupSize() / groupSize; - cpu = cpuPerGroup * current.groups() / groups; + double queryCpuPerGroup = fixedCpuCostFraction * target.nodeCpu() + + (1 - fixedCpuCostFraction) * target.nodeCpu() * current.groupSize() / groupSize; + double queryCpu = queryCpuPerGroup * current.groups() / groups; + double writeCpu = target.nodeCpu() * current.groupSize() / groupSize; + cpu = clusterModel.queryCpuFraction() * queryCpu + (1 - clusterModel.queryCpuFraction()) * writeCpu; memory = target.nodeMemory() * current.groupSize() / groupSize; disk = target.nodeDisk() * current.groupSize() / groupSize; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 9791aabf7b4..2a51a921a9f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -59,7 +59,12 @@ public class Autoscaler { } private Advice autoscale(Application application, Cluster cluster, NodeList clusterNodes, Limits limits) { - ClusterModel clusterModel = new ClusterModel(application, cluster, clusterNodes, metricsDb, nodeRepository.clock()); + ClusterModel clusterModel = new ClusterModel(application, + cluster, + clusterNodes.clusterSpec(), + clusterNodes, + metricsDb, + nodeRepository.clock()); if ( ! clusterIsStable(clusterNodes, nodeRepository)) return Advice.none("Cluster change in progress"); @@ -80,7 +85,7 @@ public class Autoscaler { var target = ResourceTarget.idealLoad(clusterModel, currentAllocation); Optional<AllocatableClusterResources> bestAllocation = - allocationOptimizer.findBestAllocation(target, currentAllocation, limits); + allocationOptimizer.findBestAllocation(target, currentAllocation, clusterModel, limits); if (bestAllocation.isEmpty()) return Advice.dontScale("No allocation improvements are possible within configured limits"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 4fb91e8592e..acf227e3de2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; @@ -25,6 +26,7 @@ public class ClusterModel { private final Application application; private final Cluster cluster; + /** The current nodes of this cluster, or empty if this models a new cluster not yet deployed */ private final NodeList nodes; private final MetricsDb metricsDb; private final Clock clock; @@ -38,6 +40,7 @@ public class ClusterModel { public ClusterModel(Application application, Cluster cluster, + ClusterSpec clusterSpec, NodeList clusterNodes, MetricsDb metricsDb, Clock clock) { @@ -46,7 +49,7 @@ public class ClusterModel { this.nodes = clusterNodes; this.metricsDb = metricsDb; this.clock = clock; - this.scalingDuration = computeScalingDuration(cluster, clusterNodes); + this.scalingDuration = computeScalingDuration(cluster, clusterSpec); } /** For testing */ @@ -132,20 +135,21 @@ public class ClusterModel { (1 - queryCpuFraction) * idealWriteCpuLoad; } - private double queryCpuFraction() { + /** The estimated fraction of cpu usage which goes to processing queries vs. writes */ + public double queryCpuFraction() { OptionalDouble queryRate = clusterTimeseries().queryRate(scalingDuration(), clock); OptionalDouble writeRate = clusterTimeseries().writeRate(scalingDuration(), clock); if (queryRate.orElse(0) == 0 && writeRate.orElse(0) == 0) return queryCpuFraction(0.5); return queryCpuFraction(queryRate.orElse(0) / (queryRate.orElse(0) + writeRate.orElse(0))); } - private double queryCpuFraction(double queryFraction) { + private double queryCpuFraction(double queryRateFraction) { double relativeQueryCost = 9; // How much more expensive are queries than writes? TODO: Measure - double writeFraction = 1 - queryFraction; - return queryFraction * relativeQueryCost / (queryFraction * relativeQueryCost + writeFraction); + double writeFraction = 1 - queryRateFraction; + return queryRateFraction * relativeQueryCost / (queryRateFraction * relativeQueryCost + writeFraction); } - private static Duration computeScalingDuration(Cluster cluster, NodeList nodes) { + private static Duration computeScalingDuration(Cluster cluster, ClusterSpec clusterSpec) { int completedEventCount = 0; Duration totalDuration = Duration.ZERO; for (ScalingEvent event : cluster.scalingEvents()) { @@ -155,14 +159,14 @@ public class ClusterModel { } if (completedEventCount == 0) { // Use defaults - if (nodes.clusterSpec().isStateful()) return Duration.ofHours(12); + if (clusterSpec.isStateful()) return Duration.ofHours(12); return Duration.ofMinutes(10); } else { Duration predictedDuration = totalDuration.dividedBy(completedEventCount); // TODO: Remove when we have reliable completion for content clusters - if (nodes.clusterSpec().isStateful() && predictedDuration.minus(Duration.ofHours(12)).isNegative()) + if (clusterSpec.isStateful() && predictedDuration.minus(Duration.ofHours(12)).isNegative()) return Duration.ofHours(12); if (predictedDuration.minus(Duration.ofMinutes(5)).isNegative()) return Duration.ofMinutes(5); // minimum diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java index c5a2bc97ef2..7a29414642e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -11,7 +12,11 @@ import java.time.Duration; import java.util.List; /** - * This moves nodes from provisioned to parked if they have been in provisioned too long. + * This moves nodes of type {@link NodeType#host} from preovisioned to parked if they have been in provisioned too long. + * + * Only {@link NodeType#host} is moved because any number of nodes of that type can exist. Other node types such as + * {@link NodeType#confighost} have a fixed number and thus cannot be replaced while the fixed number of nodes exist in + * any state. * * @author freva */ @@ -23,8 +28,11 @@ public class ProvisionedExpirer extends Expirer { @Override protected void expire(List<Node> expired) { - for (Node expiredNode : expired) - nodeRepository().nodes().parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned"); + for (Node expiredNode : expired) { + if (expiredNode.type() == NodeType.host) { + nodeRepository().nodes().parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned"); + } + } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/JobControlFlags.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/JobControlFlags.java index 79f4403e534..bea3240a0e8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/JobControlFlags.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/JobControlFlags.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.concurrent.maintenance.JobControlState; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index b286cebce6a..75fa3aec1e2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -19,11 +19,15 @@ import com.yahoo.config.provision.Zone; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.AllocationOptimizer; +import com.yahoo.vespa.hosted.provision.autoscale.ClusterModel; import com.yahoo.vespa.hosted.provision.autoscale.Limits; +import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.autoscale.ResourceTarget; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; @@ -48,6 +52,7 @@ public class NodeRepositoryProvisioner implements Provisioner { private static final Logger log = Logger.getLogger(NodeRepositoryProvisioner.class.getName()); private final NodeRepository nodeRepository; + private final MetricsDb metricsDb; private final AllocationOptimizer allocationOptimizer; private final CapacityPolicies capacityPolicies; private final Zone zone; @@ -57,9 +62,12 @@ public class NodeRepositoryProvisioner implements Provisioner { private final NodeResourceLimits nodeResourceLimits; @Inject - public NodeRepositoryProvisioner(NodeRepository nodeRepository, Zone zone, + public NodeRepositoryProvisioner(NodeRepository nodeRepository, + MetricsDb metricsDb, + Zone zone, ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource) { this.nodeRepository = nodeRepository; + this.metricsDb = metricsDb; this.allocationOptimizer = new AllocationOptimizer(nodeRepository); this.capacityPolicies = new CapacityPolicies(nodeRepository); this.zone = zone; @@ -137,35 +145,37 @@ public class NodeRepositoryProvisioner implements Provisioner { */ private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity requested) { try (Mutex lock = nodeRepository.nodes().lock(applicationId)) { - Application application = nodeRepository.applications().get(applicationId).orElse(Application.empty(applicationId)); - application = application.withCluster(clusterSpec.id(), clusterSpec.isExclusive(), requested.minResources(), requested.maxResources()); + var application = nodeRepository.applications().get(applicationId).orElse(Application.empty(applicationId)) + .withCluster(clusterSpec.id(), clusterSpec.isExclusive(), requested.minResources(), requested.maxResources()); nodeRepository.applications().put(application, lock); - return application.clusters().get(clusterSpec.id()).targetResources() - .orElseGet(() -> currentResources(applicationId, clusterSpec, requested)); + var cluster = application.cluster(clusterSpec.id()).get(); + return cluster.targetResources().orElseGet(() -> currentResources(application, clusterSpec, cluster, requested)); } } /** Returns the current resources of this cluster, or requested min if none */ - private ClusterResources currentResources(ApplicationId applicationId, + private ClusterResources currentResources(Application application, ClusterSpec clusterSpec, + Cluster cluster, Capacity requested) { - List<Node> nodes = nodeRepository.nodes().list(Node.State.active).owner(applicationId) - .cluster(clusterSpec.id()) - .not().retired() - .not().removable() - .asList(); + NodeList nodes = nodeRepository.nodes().list(Node.State.active).owner(application.id()) + .cluster(clusterSpec.id()) + .not().retired() + .not().removable(); boolean firstDeployment = nodes.isEmpty(); AllocatableClusterResources currentResources = firstDeployment // start at min, preserve current resources otherwise ? new AllocatableClusterResources(requested.minResources(), clusterSpec, nodeRepository) - : new AllocatableClusterResources(nodes, nodeRepository, clusterSpec.isExclusive()); - return within(Limits.of(requested), currentResources, firstDeployment); + : new AllocatableClusterResources(nodes.asList(), nodeRepository, clusterSpec.isExclusive()); + var clusterModel = new ClusterModel(application, cluster, clusterSpec, nodes, metricsDb, nodeRepository.clock()); + return within(Limits.of(requested), currentResources, firstDeployment, clusterModel); } /** Make the minimal adjustments needed to the current resources to stay within the limits */ private ClusterResources within(Limits limits, AllocatableClusterResources current, - boolean firstDeployment) { + boolean firstDeployment, + ClusterModel clusterModel) { if (limits.min().equals(limits.max())) return limits.min(); // Don't change current deployments that are still legal @@ -173,7 +183,10 @@ public class NodeRepositoryProvisioner implements Provisioner { if (! firstDeployment && currentAsAdvertised.isWithin(limits.min(), limits.max())) return currentAsAdvertised; // Otherwise, find an allocation that preserves the current resources as well as possible - return allocationOptimizer.findBestAllocation(ResourceTarget.preserve(current), current, limits) + return allocationOptimizer.findBestAllocation(ResourceTarget.preserve(current), + current, + clusterModel, + limits) .orElseThrow(() -> new IllegalArgumentException("No allocation possible within " + limits)) .advertisedResources(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index cc59860384b..176bf195f1f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -69,7 +69,7 @@ public class ApplicationSerializer { NodeList nodes = applicationNodes.not().retired().cluster(cluster.id()); if (nodes.isEmpty()) return; ClusterResources currentResources = nodes.toResources(); - ClusterModel clusterModel = new ClusterModel(application, cluster, nodes, metricsDb, nodeRepository.clock()); + ClusterModel clusterModel = new ClusterModel(application, cluster, nodes.clusterSpec(), nodes, metricsDb, nodeRepository.clock()); Cursor clusterObject = clustersObject.setObject(cluster.id().value()); clusterObject.setString("type", nodes.clusterSpec().type().name()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index f6649b44c0b..32952eeb860 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.autoscale.MemoryMetricsDb; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Status; @@ -77,6 +78,7 @@ public class MockNodeRepository extends NodeRepository { private void populate() { NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, + new MemoryMetricsDb(this), Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java index 60eb66c1779..b812a547ede 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; import com.yahoo.component.Version; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 17ae36b3636..59f79ceab45 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -330,7 +330,7 @@ public class AutoscalingTest { } @Test - public void test_autoscaling_groupsize_by_cpu() { + public void test_autoscaling_groupsize_by_cpu_read_dominated() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1)); @@ -343,9 +343,34 @@ public class AutoscalingTest { tester.deploy(application1, cluster1, 6, 2, resources); tester.addCpuMeasurements(0.25f, 1f, 120, application1); tester.clock().advance(Duration.ofMinutes(-10 * 5)); - tester.addQueryRateMeasurements(application1, cluster1.id(), 10, t -> t == 0 ? 20.0 : 10.0); // Query traffic only + tester.addLoadMeasurements(application1, cluster1.id(), 10, + t -> t == 0 ? 20.0 : 10.0, + t -> 1.0); tester.assertResources("Scaling up since resource usage is too high, changing to 1 group is cheaper", - 8, 1, 2.7, 83.3, 83.3, + 8, 1, 2.6, 83.3, 83.3, + tester.autoscale(application1, cluster1.id(), min, max).target()); + } + + /** Same as above but mostly write traffic, which favors smaller groups */ + @Test + public void test_autoscaling_groupsize_by_cpu_write_dominated() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1)); + AutoscalingTester tester = new AutoscalingTester(resources.withVcpu(resources.vcpu() * 2)); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 6, 2, resources); + tester.addCpuMeasurements(0.25f, 1f, 120, application1); + tester.clock().advance(Duration.ofMinutes(-10 * 5)); + tester.addLoadMeasurements(application1, cluster1.id(), 10, + t -> t == 0 ? 20.0 : 10.0, + t -> 100.0); + tester.assertResources("Scaling down since resource usage is too high, changing to 1 group is cheaper", + 4, 1, 2.1, 83.3, 83.3, tester.autoscale(application1, cluster1.id(), min, max).target()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java index 6f50cbf9803..cbfaaeaf61c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; +import com.yahoo.vespa.hosted.provision.autoscale.MemoryMetricsDb; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -60,8 +61,11 @@ public class InfraDeployerImplTest { private final NodeRepositoryTester tester = new NodeRepositoryTester(); private final NodeRepository nodeRepository = tester.nodeRepository(); - private final Provisioner provisioner = spy(new NodeRepositoryProvisioner( - nodeRepository, Zone.defaultZone(), new EmptyProvisionServiceProvider(), new InMemoryFlagSource())); + private final Provisioner provisioner = spy(new NodeRepositoryProvisioner(nodeRepository, + new MemoryMetricsDb(nodeRepository), + Zone.defaultZone(), + new EmptyProvisionServiceProvider(), + new InMemoryFlagSource())); private final InfrastructureVersions infrastructureVersions = nodeRepository.infrastructureVersions(); private final DuperModelInfraApi duperModelInfraApi = mock(DuperModelInfraApi.class); private final InfraDeployerImpl infraDeployer; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 3e71b7f5158..00d87bc448e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -35,6 +35,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.Nodelike; +import com.yahoo.vespa.hosted.provision.autoscale.MemoryMetricsDb; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceMock; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; @@ -115,7 +116,11 @@ public class ProvisioningTester { spareCount, 1000); this.orchestrator = orchestrator; - this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource); + this.provisioner = new NodeRepositoryProvisioner(nodeRepository, + new MemoryMetricsDb(nodeRepository), + zone, + provisionServiceProvider, + flagSource); this.capacityPolicies = new CapacityPolicies(nodeRepository); this.provisionLogger = new NullProvisionLogger(); this.loadBalancerService = loadBalancerService; diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java index 39c93291bad..6b656f73305 100644 --- a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java +++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.restapi.wire; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/SuspensionReasonsTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/SuspensionReasonsTest.java index 11274dc5d75..10290e5c814 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/SuspensionReasonsTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/SuspensionReasonsTest.java @@ -1,4 +1,5 @@ -package com.yahoo.vespa.orchestrator.policy;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.policy; import com.yahoo.test.ManualClock; import com.yahoo.vespa.applicationmodel.HostName; @@ -37,4 +38,4 @@ public class SuspensionReasonsTest { } -}
\ No newline at end of file +} diff --git a/persistence/src/vespa/persistence/spi/bucket_limits.h b/persistence/src/vespa/persistence/spi/bucket_limits.h index 65b2660eb23..81c837da747 100644 --- a/persistence/src/vespa/persistence/spi/bucket_limits.h +++ b/persistence/src/vespa/persistence/spi/bucket_limits.h @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once #include <cstdint> diff --git a/searchcore/src/apps/vespa-feed-bm/runtest.sh b/searchcore/src/apps/vespa-feed-bm/runtest.sh index 123cce6fdf0..adf299d424c 100755 --- a/searchcore/src/apps/vespa-feed-bm/runtest.sh +++ b/searchcore/src/apps/vespa-feed-bm/runtest.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. numdocs=500000 stripe_bits=8 diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index 5f45849e3e0..85c12acb57d 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -430,6 +430,10 @@ struct LoaderTestValues { } }; +template <> std::vector<int32_t> LoaderTestValues<NumericEnumStore>::values{3, 5, 7, 9}; +template <> std::vector<float> LoaderTestValues<FloatEnumStore>::values{3.1, 5.2, 7.3, 9.4}; +template <> std::vector<const char *> LoaderTestValues<StringEnumStore>::values{"aa", "bbb", "ccc", "dd"}; + template <> void LoaderTestValues<StringEnumStore>::load_values(enumstore::EnumeratedLoaderBase& loader) @@ -444,10 +448,6 @@ LoaderTestValues<StringEnumStore>::load_values(enumstore::EnumeratedLoaderBase& loader.load_unique_values(raw_values.data(), raw_values.size()); } -template <> std::vector<int32_t> LoaderTestValues<NumericEnumStore>::values{3, 5, 7, 9}; -template <> std::vector<float> LoaderTestValues<FloatEnumStore>::values{3.1, 5.2, 7.3, 9.4}; -template <> std::vector<const char *> LoaderTestValues<StringEnumStore>::values{"aa", "bbb", "ccc", "dd"}; - template <typename EnumStoreTypeAndOrdering> class LoaderTest : public ::testing::Test { public: diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index 34e913a4c07..6605c7ad704 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -89,6 +89,10 @@ EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::remove(const EntryCompar assert(EntryRef(itr.getData()) == EntryRef()); } this->_dict.remove(itr); + if constexpr (has_unordered_dictionary) { + auto *result = this->_unordered_dict.remove(comp, ref); + assert(result != nullptr && result->first.load_relaxed() == ref); + } } template <typename DictionaryT, typename UnorderedDictionaryT> @@ -217,6 +221,21 @@ EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::update_posting_list(Inde itr.writeData(new_posting_idx.ref()); } +template <typename DictionaryT, typename UnorderedDictionaryT> +void +EnumStoreDictionary<DictionaryT, UnorderedDictionaryT>::sync_unordered_after_load() +{ + if constexpr (has_unordered_dictionary) { + for (auto itr = this->_dict.begin(); itr.valid(); ++itr) { + EntryRef ref(itr.getKey()); + std::function<EntryRef(void)> insert_unordered_entry([ref]() noexcept -> EntryRef { return ref; }); + auto& add_result = this->_unordered_dict.add(this->_unordered_dict.get_default_comparator(), ref, insert_unordered_entry); + assert(add_result.first.load_relaxed() == ref); + add_result.second.store_relaxed(EntryRef(itr.getData())); + } + } +} + template <> EnumPostingTree & EnumStoreDictionary<EnumTree>::get_posting_dictionary() @@ -256,6 +275,7 @@ EnumStoreFoldedDictionary::~EnumStoreFoldedDictionary() = default; UniqueStoreAddResult EnumStoreFoldedDictionary::add(const EntryComparator& comp, std::function<EntryRef(void)> insertEntry) { + static_assert(!has_unordered_dictionary, "Folded Dictionary does not support unordered"); auto it = _dict.lowerBound(EntryRef(), comp); if (it.valid() && !comp.less(EntryRef(), it.getKey())) { // Entry already exists @@ -279,6 +299,7 @@ EnumStoreFoldedDictionary::add(const EntryComparator& comp, std::function<EntryR void EnumStoreFoldedDictionary::remove(const EntryComparator& comp, EntryRef ref) { + static_assert(!has_unordered_dictionary, "Folded Dictionary does not support unordered"); assert(ref.valid()); auto it = _dict.lowerBound(ref, comp); assert(it.valid() && it.getKey() == ref); diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h index 63ccb2efac7..2c2c39451c0 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h @@ -24,7 +24,9 @@ private: using IndexVector = IEnumStoreDictionary::IndexVector; using ParentUniqueStoreDictionary = vespalib::datastore::UniqueStoreDictionary<DictionaryT, IEnumStoreDictionary, UnorderedDictionaryT>; using generation_t = IEnumStoreDictionary::generation_t; - +protected: + using ParentUniqueStoreDictionary::has_unordered_dictionary; +private: IEnumStore& _enumStore; void remove_unused_values(const IndexSet& unused, @@ -56,6 +58,7 @@ public: Index remap_index(Index idx) override; void clear_all_posting_lists(std::function<void(EntryRef)> clearer) override; void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) override; + void sync_unordered_after_load() override; EnumPostingTree& get_posting_dictionary() override; const EnumPostingTree& get_posting_dictionary() const override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h index 968d1fa8747..309f037ac84 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h @@ -53,6 +53,7 @@ public: virtual Index remap_index(Index idx) = 0; virtual void clear_all_posting_lists(std::function<void(EntryRef)> clearer) = 0; virtual void update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) = 0; + virtual void sync_unordered_after_load() = 0; virtual EnumPostingTree& get_posting_dictionary() = 0; virtual const EnumPostingTree& get_posting_dictionary() const = 0; }; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp index d3223901a71..c76571b151d 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp @@ -102,6 +102,7 @@ PostingListAttributeBase<P>::handle_load_posting_lists_and_update_enum_store(enu &postings._removals[0] + postings._removals.size()); posting_itr.writeData(newIndex.ref()); loader.free_unused_values(); + _dictionary.sync_unordered_after_load(); } template <typename P> diff --git a/searchlib/src/vespa/searchlib/queryeval/elementiterator.cpp b/searchlib/src/vespa/searchlib/queryeval/elementiterator.cpp index da4a088adea..2533d602a1e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/elementiterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/elementiterator.cpp @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "elementiterator.h" #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/objectvisitor.h> diff --git a/searchlib/src/vespa/searchlib/queryeval/elementiterator.h b/searchlib/src/vespa/searchlib/queryeval/elementiterator.h index a433c9be8f4..d58d981e63b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/elementiterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/elementiterator.h @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once #include <vespa/searchlib/queryeval/searchiterator.h> diff --git a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp index af3ce5fe64f..76257a68436 100644 --- a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp @@ -1,4 +1,4 @@ -// Copyright 2020 Oath inc.. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "isequencedtaskexecutor.h" #include <vespa/vespalib/stllike/hash_fun.h> diff --git a/tenant-cd-commons/README.md b/tenant-cd-commons/README.md index b1cd95606c8..d627be59fa6 100644 --- a/tenant-cd-commons/README.md +++ b/tenant-cd-commons/README.md @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # tenant-cd-commons -Contains shared implementations of APIs/interfaces of `tenant-cd-api`.
\ No newline at end of file +Contains shared implementations of APIs/interfaces of `tenant-cd-api`. diff --git a/travis/detect-what-to-build.sh b/travis/detect-what-to-build.sh index 5fb18093037..47cb8ccfa00 100755 --- a/travis/detect-what-to-build.sh +++ b/travis/detect-what-to-build.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. if (( ${#BASH_SOURCE[@]} == 1 )); then echo "This script must be sourced." diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/ErrorHandler.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/ErrorHandler.java index e5f186f2a80..19e55d3a680 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/ErrorHandler.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/ErrorHandler.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.client; public interface ErrorHandler { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/package-info.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/package-info.java index 5efb1a89130..7c152d3e88a 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/package-info.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/package-info.java @@ -1,5 +1,6 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package com.yahoo.vespa.athenz.client; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespa-maven-plugin/src/test/resources/effective-services/dev.xml b/vespa-maven-plugin/src/test/resources/effective-services/dev.xml index d6343e7ace7..c3a863dfc63 100644 --- a/vespa-maven-plugin/src/test/resources/effective-services/dev.xml +++ b/vespa-maven-plugin/src/test/resources/effective-services/dev.xml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> -<services xmlns:deploy="vespa"> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --><services xmlns:deploy="vespa"> <container> <component id="good-service-client"> <config> diff --git a/vespa-maven-plugin/src/test/resources/effective-services/prod_us-east-3.xml b/vespa-maven-plugin/src/test/resources/effective-services/prod_us-east-3.xml index 0bab4b7e74a..ce5459b71b7 100644 --- a/vespa-maven-plugin/src/test/resources/effective-services/prod_us-east-3.xml +++ b/vespa-maven-plugin/src/test/resources/effective-services/prod_us-east-3.xml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> -<services xmlns:deploy="vespa"> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --><services xmlns:deploy="vespa"> <container> <component id="good-service-client"> <config> diff --git a/vespa-maven-plugin/src/test/resources/effective-services/prod_us-west-1.xml b/vespa-maven-plugin/src/test/resources/effective-services/prod_us-west-1.xml index e29ba231092..66b8073c7af 100644 --- a/vespa-maven-plugin/src/test/resources/effective-services/prod_us-west-1.xml +++ b/vespa-maven-plugin/src/test/resources/effective-services/prod_us-west-1.xml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> -<services xmlns:deploy="vespa"> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --><services xmlns:deploy="vespa"> <container> <component id="good-service-client"> <config> diff --git a/vespa-maven-plugin/src/test/resources/effective-services/services.xml b/vespa-maven-plugin/src/test/resources/effective-services/services.xml index 184eb2ab48c..fb41d8adf9d 100644 --- a/vespa-maven-plugin/src/test/resources/effective-services/services.xml +++ b/vespa-maven-plugin/src/test/resources/effective-services/services.xml @@ -1,4 +1,5 @@ <?xml version='1.0' encoding='UTF-8'?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services xmlns:deploy="vespa"> <container> <component id="good-service-client"> diff --git a/vespa-maven-plugin/src/test/resources/effective-services/test_us-east-1.xml b/vespa-maven-plugin/src/test/resources/effective-services/test_us-east-1.xml index 7f9e1bb93b9..93f94751907 100644 --- a/vespa-maven-plugin/src/test/resources/effective-services/test_us-east-1.xml +++ b/vespa-maven-plugin/src/test/resources/effective-services/test_us-east-1.xml @@ -1,4 +1,5 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services xmlns:deploy="vespa"> <container> <component id="good-service-client"> diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java index ab7b224c7be..c1cb368cdbd 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java @@ -1,5 +1,5 @@ -package com.yahoo.vespa.http.server;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; import com.yahoo.container.handler.threadpool.ContainerThreadPool; import com.yahoo.container.jdisc.RequestHandlerTestDriver; import com.yahoo.document.config.DocumentmanagerConfig; diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/CompletableFuturesTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/CompletableFuturesTest.java index cf9c36537d9..c2e82c1085f 100644 --- a/vespajlib/src/test/java/com/yahoo/concurrent/CompletableFuturesTest.java +++ b/vespajlib/src/test/java/com/yahoo/concurrent/CompletableFuturesTest.java @@ -1,4 +1,5 @@ -package com.yahoo.concurrent;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; import org.junit.jupiter.api.Test; @@ -63,4 +64,4 @@ class CompletableFuturesTest { } } -}
\ No newline at end of file +} diff --git a/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp b/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp index 2f5e5b9295d..0fad98af10b 100644 --- a/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp +++ b/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp @@ -108,7 +108,7 @@ DataStoreFixedSizeHashTest::insert(uint32_t key) { MyCompare comp(_store, key); std::function<EntryRef(void)> insert_entry([this, key]() -> EntryRef { return _allocator.allocate(key); }); - auto& result = _hash_map->add(comp, insert_entry); + auto& result = _hash_map->add(comp, EntryRef(), insert_entry); auto ref = result.first.load_relaxed(); auto &wrapped_entry = _allocator.get_wrapped(ref); EXPECT_EQ(key, wrapped_entry.value()); diff --git a/vespalib/src/tests/datastore/simple_hash_map/simple_hash_map_test.cpp b/vespalib/src/tests/datastore/simple_hash_map/simple_hash_map_test.cpp index e89a0aab038..4dbb35d8516 100644 --- a/vespalib/src/tests/datastore/simple_hash_map/simple_hash_map_test.cpp +++ b/vespalib/src/tests/datastore/simple_hash_map/simple_hash_map_test.cpp @@ -102,8 +102,8 @@ void DataStoreSimpleHashTest::insert(uint32_t key) { MyCompare comp(_store, key); -std::function<EntryRef(void)> insert_entry([this, key]() -> EntryRef { return _allocator.allocate(key); }); - auto& result = _hash_map.add(comp, insert_entry); + std::function<EntryRef(void)> insert_entry([this, key]() -> EntryRef { return _allocator.allocate(key); }); + auto& result = _hash_map.add(comp, EntryRef(), insert_entry); auto ref = result.first.load_relaxed(); auto &wrapped_entry = _allocator.get_wrapped(ref); EXPECT_EQ(key, wrapped_entry.value()); diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index fad78ea30ae..2e1eea55fe1 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/datastore/unique_store_remapper.h> #include <vespa/vespalib/datastore/unique_store_string_allocator.hpp> #include <vespa/vespalib/datastore/unique_store_string_comparator.h> +#include <vespa/vespalib/datastore/simple_hash_map.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/test/datastore/buffer_stats.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -12,17 +13,27 @@ #include <vespa/log/log.h> LOG_SETUP("unique_store_test"); +enum class Ordering { ORDERED, UNORDERED }; + using namespace vespalib::datastore; using vespalib::ArrayRef; using generation_t = vespalib::GenerationHandler::generation_t; using vespalib::datastore::test::BufferStats; template <typename UniqueStoreT> -struct TestBase : public ::testing::Test { - using EntryRefType = typename UniqueStoreT::RefType; +struct TestBaseValues { using UniqueStoreType = UniqueStoreT; - using ValueType = typename UniqueStoreT::EntryType; - using ValueConstRefType = typename UniqueStoreT::EntryConstRefType; + using ValueType = typename UniqueStoreType::EntryType; + static std::vector<ValueType> values; +}; + +template <typename UniqueStoreTypeAndOrder> +struct TestBase : public ::testing::Test { + using UniqueStoreType = typename UniqueStoreTypeAndOrder::UniqueStoreType; + using EntryRefType = typename UniqueStoreType::RefType; + using ValueType = typename UniqueStoreType::EntryType; + using ValueConstRefType = typename UniqueStoreType::EntryConstRefType; + using CompareType = typename UniqueStoreType::CompareType; using ReferenceStoreValueType = std::conditional_t<std::is_same_v<ValueType, const char *>, std::string, ValueType>; using ReferenceStore = std::map<EntryRef, std::pair<ReferenceStoreValueType,uint32_t>>; @@ -30,10 +41,9 @@ struct TestBase : public ::testing::Test { ReferenceStore refStore; generation_t generation; - static std::vector<ValueType> values; - TestBase(); ~TestBase() override; + const std::vector<ValueType>& values() const noexcept { return TestBaseValues<UniqueStoreType>::values; } void assertAdd(ValueConstRefType input) { EntryRef ref = add(input); assertGet(ref, input); @@ -132,16 +142,22 @@ struct TestBase : public ::testing::Test { } }; -template <typename UniqueStoreT> -TestBase<UniqueStoreT>::TestBase() +template <typename UniqueStoreTypeAndOrder> +TestBase<UniqueStoreTypeAndOrder>::TestBase() : store(), refStore(), generation(1) { + switch (UniqueStoreTypeAndOrder::ordering) { + case Ordering::ORDERED: + break; + default: + store.set_dictionary(std::make_unique<UniqueStoreDictionary<uniquestore::DefaultDictionary, IUniqueStoreDictionary, SimpleHashMap>>(std::make_unique<CompareType>(store.get_data_store()))); + } } -template <typename UniqueStoreT> -TestBase<UniqueStoreT>::~TestBase() = default; +template <typename UniqueStoreTypeAndOrder> +TestBase<UniqueStoreTypeAndOrder>::~TestBase() = default; using NumberUniqueStore = UniqueStore<uint32_t>; using StringUniqueStore = UniqueStore<std::string>; @@ -150,15 +166,75 @@ using DoubleUniqueStore = UniqueStore<double>; using SmallOffsetNumberUniqueStore = UniqueStore<uint32_t, EntryRefT<10,10>>; template <> -std::vector<uint32_t> TestBase<NumberUniqueStore>::values{10, 20, 30, 10 }; +std::vector<uint32_t> TestBaseValues<NumberUniqueStore>::values{10, 20, 30, 10 }; template <> -std::vector<std::string> TestBase<StringUniqueStore>::values{ "aa", "bbb", "ccc", "aa" }; +std::vector<std::string> TestBaseValues<StringUniqueStore>::values{ "aa", "bbb", "ccc", "aa" }; template <> -std::vector<const char *> TestBase<CStringUniqueStore>::values{ "aa", "bbb", "ccc", "aa" }; +std::vector<const char *> TestBaseValues<CStringUniqueStore>::values{ "aa", "bbb", "ccc", "aa" }; template <> -std::vector<double> TestBase<DoubleUniqueStore>::values{ 10.0, 20.0, 30.0, 10.0 }; +std::vector<double> TestBaseValues<DoubleUniqueStore>::values{ 10.0, 20.0, 30.0, 10.0 }; + +struct OrderedNumberUniqueStore +{ + using UniqueStoreType = NumberUniqueStore; + static constexpr Ordering ordering = Ordering::ORDERED; +}; + +struct OrderedStringUniqueStore +{ + using UniqueStoreType = StringUniqueStore; + static constexpr Ordering ordering = Ordering::ORDERED; +}; + +struct OrderedCStringUniqueStore +{ + using UniqueStoreType = CStringUniqueStore; + static constexpr Ordering ordering = Ordering::ORDERED; +}; + +struct OrderedDoubleUniqueStore +{ + using UniqueStoreType = DoubleUniqueStore; + static constexpr Ordering ordering = Ordering::ORDERED; +}; + +struct OrderedSmallOffsetNumberUniqueStore +{ + using UniqueStoreType = SmallOffsetNumberUniqueStore; + static constexpr Ordering ordering = Ordering::ORDERED; +}; + +struct UnorderedNumberUniqueStore +{ + using UniqueStoreType = NumberUniqueStore; + static constexpr Ordering ordering = Ordering::UNORDERED; +}; + +struct UnorderedStringUniqueStore +{ + using UniqueStoreType = StringUniqueStore; + static constexpr Ordering ordering = Ordering::UNORDERED; +}; + +struct UnorderedCStringUniqueStore +{ + using UniqueStoreType = CStringUniqueStore; + static constexpr Ordering ordering = Ordering::UNORDERED; +}; + +struct UnorderedDoubleUniqueStore +{ + using UniqueStoreType = DoubleUniqueStore; + static constexpr Ordering ordering = Ordering::UNORDERED; +}; + +struct UnorderedSmallOffsetNumberUniqueStore +{ + using UniqueStoreType = SmallOffsetNumberUniqueStore; + static constexpr Ordering ordering = Ordering::UNORDERED; +}; -using UniqueStoreTestTypes = ::testing::Types<NumberUniqueStore, StringUniqueStore, CStringUniqueStore, DoubleUniqueStore>; +using UniqueStoreTestTypes = ::testing::Types<OrderedNumberUniqueStore, OrderedStringUniqueStore, OrderedCStringUniqueStore, OrderedDoubleUniqueStore, UnorderedNumberUniqueStore, UnorderedStringUniqueStore, UnorderedCStringUniqueStore, UnorderedDoubleUniqueStore>; VESPA_GTEST_TYPED_TEST_SUITE(TestBase, UniqueStoreTestTypes); // Disable warnings emitted by gtest generated files when using typed tests @@ -167,11 +243,11 @@ VESPA_GTEST_TYPED_TEST_SUITE(TestBase, UniqueStoreTestTypes); #pragma GCC diagnostic ignored "-Wsuggest-override" #endif -using NumberTest = TestBase<NumberUniqueStore>; -using StringTest = TestBase<StringUniqueStore>; -using CStringTest = TestBase<CStringUniqueStore>; -using DoubleTest = TestBase<DoubleUniqueStore>; -using SmallOffsetNumberTest = TestBase<SmallOffsetNumberUniqueStore>; +using NumberTest = TestBase<OrderedNumberUniqueStore>; +using StringTest = TestBase<OrderedStringUniqueStore>; +using CStringTest = TestBase<OrderedCStringUniqueStore>; +using DoubleTest = TestBase<OrderedDoubleUniqueStore>; +using SmallOffsetNumberTest = TestBase<OrderedSmallOffsetNumberUniqueStore>; TEST(UniqueStoreTest, trivial_and_non_trivial_types_are_tested) { @@ -181,14 +257,14 @@ TEST(UniqueStoreTest, trivial_and_non_trivial_types_are_tested) TYPED_TEST(TestBase, can_add_and_get_values) { - for (auto &val : this->values) { + for (auto &val : this->values()) { this->assertAdd(val); } } TYPED_TEST(TestBase, elements_are_put_on_hold_when_value_is_removed) { - EntryRef ref = this->add(this->values[0]); + EntryRef ref = this->add(this->values()[0]); size_t reserved = this->get_reserved(ref); size_t array_size = this->get_array_size(ref); this->assertBufferState(ref, BufferStats().used(array_size + reserved).hold(0).dead(reserved)); @@ -198,8 +274,8 @@ TYPED_TEST(TestBase, elements_are_put_on_hold_when_value_is_removed) TYPED_TEST(TestBase, elements_are_reference_counted) { - EntryRef ref = this->add(this->values[0]); - EntryRef ref2 = this->add(this->values[0]); + EntryRef ref = this->add(this->values()[0]); + EntryRef ref2 = this->add(this->values()[0]); EXPECT_EQ(ref.ref(), ref2.ref()); // Note: The first buffer have the first element reserved -> we expect 2 elements used here. size_t reserved = this->get_reserved(ref); @@ -232,9 +308,9 @@ TEST_F(SmallOffsetNumberTest, new_underlying_buffer_is_allocated_when_current_is TYPED_TEST(TestBase, store_can_be_compacted) { - EntryRef val0Ref = this->add(this->values[0]); - EntryRef val1Ref = this->add(this->values[1]); - this->remove(this->add(this->values[2])); + EntryRef val0Ref = this->add(this->values()[0]); + EntryRef val1Ref = this->add(this->values()[1]); + this->remove(this->add(this->values()[2])); this->trimHoldLists(); size_t reserved = this->get_reserved(val0Ref); size_t array_size = this->get_array_size(val0Ref); @@ -247,10 +323,10 @@ TYPED_TEST(TestBase, store_can_be_compacted) this->assertStoreContent(); // Buffer has been compacted - EXPECT_NE(val1BufferId, this->getBufferId(this->getEntryRef(this->values[0]))); + EXPECT_NE(val1BufferId, this->getBufferId(this->getEntryRef(this->values()[0]))); // Old ref should still point to data. - this->assertGet(val0Ref, this->values[0]); - this->assertGet(val1Ref, this->values[1]); + this->assertGet(val0Ref, this->values()[0]); + this->assertGet(val1Ref, this->values()[1]); EXPECT_TRUE(this->store.bufferState(val0Ref).isOnHold()); this->trimHoldLists(); EXPECT_TRUE(this->store.bufferState(val0Ref).isFree()); @@ -260,8 +336,8 @@ TYPED_TEST(TestBase, store_can_be_compacted) TYPED_TEST(TestBase, store_can_be_instantiated_with_builder) { auto builder = this->getBuilder(2); - builder.add(this->values[0]); - builder.add(this->values[1]); + builder.add(this->values()[0]); + builder.add(this->values()[1]); builder.setupRefCounts(); EntryRef val0Ref = builder.mapEnumValueToEntryRef(1); EntryRef val1Ref = builder.mapEnumValueToEntryRef(2); @@ -271,21 +347,21 @@ TYPED_TEST(TestBase, store_can_be_instantiated_with_builder) EXPECT_TRUE(val0Ref.valid()); EXPECT_TRUE(val1Ref.valid()); EXPECT_NE(val0Ref.ref(), val1Ref.ref()); - this->assertGet(val0Ref, this->values[0]); - this->assertGet(val1Ref, this->values[1]); + this->assertGet(val0Ref, this->values()[0]); + this->assertGet(val1Ref, this->values()[1]); builder.makeDictionary(); // Align refstore with the two entries added by builder. - this->alignRefStore(val0Ref, this->values[0], 1); - this->alignRefStore(val1Ref, this->values[1], 1); - EXPECT_EQ(val0Ref.ref(), this->add(this->values[0]).ref()); - EXPECT_EQ(val1Ref.ref(), this->add(this->values[1]).ref()); + this->alignRefStore(val0Ref, this->values()[0], 1); + this->alignRefStore(val1Ref, this->values()[1], 1); + EXPECT_EQ(val0Ref.ref(), this->add(this->values()[0]).ref()); + EXPECT_EQ(val1Ref.ref(), this->add(this->values()[1]).ref()); } TYPED_TEST(TestBase, store_can_be_enumerated) { - EntryRef val0Ref = this->add(this->values[0]); - EntryRef val1Ref = this->add(this->values[1]); - this->remove(this->add(this->values[2])); + EntryRef val0Ref = this->add(this->values()[0]); + EntryRef val1Ref = this->add(this->values()[1]); + this->remove(this->add(this->values()[2])); this->trimHoldLists(); auto enumerator = this->getEnumerator(); diff --git a/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h b/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h index 3ec2d6b163e..f4724e563ca 100644 --- a/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h +++ b/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h @@ -34,6 +34,9 @@ public: void store_release(EntryRef ref) noexcept { _ref.store(ref.ref(), std::memory_order_release); } + void store_relaxed(EntryRef ref) noexcept { + _ref.store(ref.ref(), std::memory_order_relaxed); + } EntryRef load_acquire() const noexcept { return EntryRef(_ref.load(std::memory_order_acquire)); } diff --git a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp index 14dbc841691..be67e0b1863 100644 --- a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp +++ b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp @@ -62,15 +62,15 @@ FixedSizeHashMap::force_add(const EntryComparator& comp, const KvType& kv) } FixedSizeHashMap::KvType& -FixedSizeHashMap::add(const EntryComparator& comp, std::function<EntryRef(void)>& insert_entry) +FixedSizeHashMap::add(const EntryComparator& comp, EntryRef key_ref, std::function<EntryRef(void)>& insert_entry) { - size_t hash_idx = comp.hash(EntryRef()) / _num_stripes; + size_t hash_idx = comp.hash(key_ref) / _num_stripes; hash_idx %= _modulo; auto& chain_head = _chain_heads[hash_idx]; uint32_t node_idx = chain_head.load_relaxed(); while (node_idx != no_node_idx) { auto& node = _nodes[node_idx]; - if (comp.equal(EntryRef(), node.get_kv().first.load_relaxed())) { + if (comp.equal(key_ref, node.get_kv().first.load_relaxed())) { return node.get_kv(); } node_idx = node.get_next_node_idx().load(std::memory_order_relaxed); @@ -153,8 +153,8 @@ FixedSizeHashMap::remove(const EntryComparator& comp, EntryRef key_ref) return nullptr; } -const FixedSizeHashMap::KvType* -FixedSizeHashMap::find(const EntryComparator& comp, EntryRef key_ref) const +FixedSizeHashMap::KvType* +FixedSizeHashMap::find(const EntryComparator& comp, EntryRef key_ref) { size_t hash_idx = comp.hash(key_ref) / _num_stripes; hash_idx %= _modulo; diff --git a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h index 326c3cef765..72373f909df 100644 --- a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h +++ b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h @@ -96,9 +96,9 @@ public: FixedSizeHashMap(uint32_t module, uint32_t capacity, uint32_t num_stripes, const FixedSizeHashMap &orig, const EntryComparator& comp); ~FixedSizeHashMap(); - KvType& add(const EntryComparator& comp, std::function<EntryRef(void)>& insert_entry); + KvType& add(const EntryComparator& comp, EntryRef key_ref, std::function<EntryRef(void)>& insert_entry); KvType* remove(const EntryComparator& comp, EntryRef key_ref); - const KvType* find(const EntryComparator& comp, EntryRef key_ref) const; + KvType* find(const EntryComparator& comp, EntryRef key_ref); void transfer_hold_lists(generation_t generation) { if (!_hold_1_list.empty()) { diff --git a/vespalib/src/vespa/vespalib/datastore/simple_hash_map.cpp b/vespalib/src/vespa/vespalib/datastore/simple_hash_map.cpp index e397eca579a..6995acc5bb4 100644 --- a/vespalib/src/vespa/vespalib/datastore/simple_hash_map.cpp +++ b/vespalib/src/vespa/vespalib/datastore/simple_hash_map.cpp @@ -31,6 +31,7 @@ SimpleHashMap::SimpleHashMap(std::unique_ptr<const EntryComparator> comp) SimpleHashMap::~SimpleHashMap() { + _gen_holder.clearHoldLists(); for (size_t i = 0; i < num_stripes; ++i) { auto map = _maps[i].load(std::memory_order_relaxed); delete map; @@ -66,15 +67,15 @@ SimpleHashMap::hold_stripe(std::unique_ptr<const FixedSizeHashMap> map) } SimpleHashMap::KvType& -SimpleHashMap::add(const EntryComparator& comp, std::function<EntryRef(void)>& insert_entry) +SimpleHashMap::add(const EntryComparator& comp, EntryRef key_ref, std::function<EntryRef(void)>& insert_entry) { - size_t stripe = get_stripe(comp, EntryRef()); + size_t stripe = get_stripe(comp, key_ref); auto map = _maps[stripe].load(std::memory_order_relaxed); if (map == nullptr || map->full()) { alloc_stripe(stripe); map = _maps[stripe].load(std::memory_order_relaxed); } - return map->add(comp, insert_entry); + return map->add(comp, key_ref, insert_entry); } SimpleHashMap::KvType* @@ -88,6 +89,17 @@ SimpleHashMap::remove(const EntryComparator& comp, EntryRef key_ref) return map->remove(comp, key_ref); } +SimpleHashMap::KvType* +SimpleHashMap::find(const EntryComparator& comp, EntryRef key_ref) +{ + size_t stripe = get_stripe(comp, key_ref); + auto map = _maps[stripe].load(std::memory_order_relaxed); + if (map == nullptr) { + return nullptr; + } + return map->find(comp, key_ref); +} + const SimpleHashMap::KvType* SimpleHashMap::find(const EntryComparator& comp, EntryRef key_ref) const { diff --git a/vespalib/src/vespa/vespalib/datastore/simple_hash_map.h b/vespalib/src/vespa/vespalib/datastore/simple_hash_map.h index 1acc11d50c8..78e224c65df 100644 --- a/vespalib/src/vespa/vespalib/datastore/simple_hash_map.h +++ b/vespalib/src/vespa/vespalib/datastore/simple_hash_map.h @@ -46,12 +46,14 @@ private: public: SimpleHashMap(std::unique_ptr<const EntryComparator> comp); ~SimpleHashMap(); - KvType& add(const EntryComparator& comp, std::function<EntryRef(void)> &insert_entry); + KvType& add(const EntryComparator& comp, EntryRef key_ref, std::function<EntryRef(void)> &insert_entry); KvType* remove(const EntryComparator& comp, EntryRef key_ref); + KvType* find(const EntryComparator& comp, EntryRef key_ref); const KvType* find(const EntryComparator& comp, EntryRef key_ref) const; void transfer_hold_lists(generation_t generation); void trim_hold_lists(generation_t first_used); size_t size() const noexcept; + const EntryComparator &get_default_comparator() const noexcept { return *_comp; } }; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.h b/vespalib/src/vespa/vespalib/datastore/unique_store.h index 0a1593f3db3..565c1ceee61 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.h @@ -35,6 +35,7 @@ public: using DataStoreType = DataStoreT<RefT>; using EntryType = EntryT; using RefType = RefT; + using CompareType = Compare; using Enumerator = UniqueStoreEnumerator<RefT>; using Builder = UniqueStoreBuilder<Allocator>; using Remapper = UniqueStoreRemapper<RefT>; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp index 713438f8419..963a2dc72a1 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -80,6 +80,9 @@ void UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::transfer_hold_lists(generation_t generation) { _dict.getAllocator().transferHoldLists(generation); + if constexpr (has_unordered_dictionary) { + this->_unordered_dict.transfer_hold_lists(generation); + } } template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> @@ -87,6 +90,9 @@ void UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::trim_hold_lists(generation_t firstUsed) { _dict.getAllocator().trimHoldLists(firstUsed); + if constexpr (has_unordered_dictionary) { + this->_unordered_dict.trim_hold_lists(firstUsed); + } } template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> @@ -96,11 +102,20 @@ UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::add(const Ent { auto itr = _dict.lowerBound(EntryRef(), comp); if (itr.valid() && !comp.less(EntryRef(), itr.getKey())) { + if constexpr (has_unordered_dictionary) { + auto* result = this->_unordered_dict.find(comp, EntryRef()); + assert(result != nullptr && result->first.load_relaxed() == itr.getKey()); + } return UniqueStoreAddResult(itr.getKey(), false); } else { EntryRef newRef = insertEntry(); _dict.insert(itr, newRef, DataType()); + if constexpr (has_unordered_dictionary) { + std::function<EntryRef(void)> insert_unordered_entry([newRef]() noexcept -> EntryRef { return newRef; }); + auto& add_result = this->_unordered_dict.add(comp, newRef, insert_unordered_entry); + assert(add_result.first.load_relaxed() == newRef); + } return UniqueStoreAddResult(newRef, true); } } @@ -111,8 +126,16 @@ UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::find(const En { auto itr = _dict.lowerBound(EntryRef(), comp); if (itr.valid() && !comp.less(EntryRef(), itr.getKey())) { + if constexpr (has_unordered_dictionary) { + auto* result = this->_unordered_dict.find(comp, EntryRef()); + assert(result != nullptr && result->first.load_relaxed() == itr.getKey()); + } return itr.getKey(); } else { + if constexpr (has_unordered_dictionary) { + auto* result = this->_unordered_dict.find(comp, EntryRef()); + assert(result == nullptr); + } return EntryRef(); } } @@ -125,6 +148,10 @@ UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::remove(const auto itr = _dict.lowerBound(ref, comp); assert(itr.valid() && itr.getKey() == ref); _dict.remove(itr); + if constexpr (has_unordered_dictionary) { + auto *result = this->_unordered_dict.remove(comp, ref); + assert(result != nullptr && result->first.load_relaxed() == ref); + } } template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> @@ -138,6 +165,11 @@ UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::move_entries( if (newRef != oldRef) { _dict.thaw(itr); itr.writeKey(newRef); + if constexpr (has_unordered_dictionary) { + auto result = this->_unordered_dict.find(this->_unordered_dict.get_default_comparator(), oldRef); + assert(result != nullptr && result->first.load_relaxed() == oldRef); + result->first.store_release(newRef); + } } ++itr; } @@ -174,6 +206,16 @@ UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::build(vespali } } _dict.assign(builder); + if constexpr (has_unordered_dictionary) { + for (size_t i = 1; i < refs.size(); ++i) { + if (ref_counts[i] != 0u) { + EntryRef ref = refs[i]; + std::function<EntryRef(void)> insert_unordered_entry([ref]() noexcept -> EntryRef { return ref; }); + auto& add_result = this->_unordered_dict.add(this->_unordered_dict.get_default_comparator(), ref, insert_unordered_entry); + assert(add_result.first.load_relaxed() == ref); + } + } + } } template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> @@ -185,6 +227,13 @@ UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::build(vespali builder.insert(ref, DataType()); } _dict.assign(builder); + if constexpr (has_unordered_dictionary) { + for (const auto& ref : refs) { + std::function<EntryRef(void)> insert_unordered_entry([ref]() noexcept -> EntryRef { return ref; }); + auto& add_result = this->_unordered_dict.add(this->_unordered_dict.get_default_comparator(), ref, insert_unordered_entry); + assert(add_result.first.load_relaxed() == ref); + } + } } template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> @@ -202,6 +251,17 @@ UniqueStoreDictionary<DictionaryT, ParentT, UnorderedDictionaryT>::build_with_pa } } _dict.assign(builder); + if constexpr (has_unordered_dictionary) { + for (size_t i = 0; i < refs.size(); ++i) { + EntryRef ref = refs[i]; + std::function<EntryRef(void)> insert_unordered_entry([ref]() noexcept -> EntryRef { return ref; }); + auto& add_result = this->_unordered_dict.add(this->_unordered_dict.get_default_comparator(), ref, insert_unordered_entry); + assert(add_result.first.load_relaxed() == refs[i]); + if constexpr (std::is_same_v<DataType, uint32_t>) { + add_result.second.store_relaxed(EntryRef(payloads[i])); + } + } + } } template <typename DictionaryT, typename ParentT, typename UnorderedDictionaryT> diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_comparator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_string_comparator.h index 9acacc0073f..90a2a76b913 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_comparator.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_comparator.h @@ -38,6 +38,12 @@ protected: } public: + UniqueStoreStringComparator(const DataStoreType &store) + : _store(store), + _fallback_value(nullptr) + { + } + UniqueStoreStringComparator(const DataStoreType &store, const char *fallback_value) : _store(store), _fallback_value(fallback_value) diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LatencyStatsTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LatencyStatsTest.java index 981b3571056..44d9e6e73ba 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LatencyStatsTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LatencyStatsTest.java @@ -1,4 +1,5 @@ -package com.yahoo.vespa.curator.stats;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.curator.stats; import com.yahoo.vespa.curator.stats.LatencyStats.ActiveInterval; import org.junit.Test; @@ -141,4 +142,4 @@ public class LatencyStatsTest { @Override public long getAsLong() { return nanoTime; } } -}
\ No newline at end of file +} diff --git a/zookeeper-client-common/README.md b/zookeeper-client-common/README.md index 51c757a8af2..9f8952a0cd3 100644 --- a/zookeeper-client-common/README.md +++ b/zookeeper-client-common/README.md @@ -1,3 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> # zookeeper-client-common Shared client configuration logic for ZooKeeper clients diff --git a/zookeeper-server/zookeeper-server-3.6.2/pom.xml b/zookeeper-server/zookeeper-server-3.6.2/pom.xml index 522bc71df97..04f8b012b09 100644 --- a/zookeeper-server/zookeeper-server-3.6.2/pom.xml +++ b/zookeeper-server/zookeeper-server-3.6.2/pom.xml @@ -18,6 +18,18 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-client-common</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- Don't use ZK version from zookeeper-client-common --> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>3.6.2</version> diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java index 8ae30942d55..d92527fb5fd 100644 --- a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.zookeeper; +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.admin.ZooKeeperAdmin; @@ -45,7 +46,7 @@ public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { private ZooKeeperAdmin createAdmin(String connectionSpec) throws IOException { return new ZooKeeperAdmin(connectionSpec, (int) sessionTimeout().toMillis(), - (event) -> log.log(Level.INFO, event.toString())); + (event) -> log.log(Level.INFO, event.toString()), new ZkClientConfigBuilder().toConfig()); } private static boolean retryOn(KeeperException e) { diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/org/apache/zookeeper/common/NetUtils.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/org/apache/zookeeper/common/NetUtils.java index f32f1da7c82..79d063ba70a 100644 --- a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/org/apache/zookeeper/common/NetUtils.java +++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/org/apache/zookeeper/common/NetUtils.java @@ -1,3 +1,4 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file diff --git a/zookeeper-server/zookeeper-server-3.6.2/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java new file mode 100644 index 00000000000..7efec454667 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.6.2/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java @@ -0,0 +1,37 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package org.apache.zookeeper.server; + +import com.yahoo.vespa.zookeeper.Configurator; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.logging.Logger; + +/** + * Overrides secure setting with value from {@link Configurator}. + * Workaround for incorrect handling of clientSecurePort in combination with ZooKeeper Dynamic Reconfiguration in 3.6.2 + * See https://issues.apache.org/jira/browse/ZOOKEEPER-3577. + * + * Using package {@link org.apache.zookeeper.server} as {@link NettyServerCnxnFactory#NettyServerCnxnFactory()} is package-private. + * + * @author bjorncs + */ +public class VespaNettyServerCnxnFactory extends NettyServerCnxnFactory { + + private static final Logger log = Logger.getLogger(VespaNettyServerCnxnFactory.class.getName()); + + private final boolean isSecure; + + public VespaNettyServerCnxnFactory() { + super(); + this.isSecure = Configurator.VespaNettyServerCnxnFactory_isSecure; + boolean portUnificationEnabled = Boolean.getBoolean(NettyServerCnxnFactory.PORT_UNIFICATION_KEY); + log.info(String.format("For %h: isSecure=%b, portUnification=%b", this, isSecure, portUnificationEnabled)); + } + + @Override + public void configure(InetSocketAddress addr, int maxClientCnxns, int backlog, boolean secure) throws IOException { + log.info(String.format("For %h: configured() invoked with parameter 'secure'=%b, overridden to %b", this, secure, isSecure)); + super.configure(addr, maxClientCnxns, backlog, isSecure); + } +} diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java index c58c97845e3..d662bab8463 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java @@ -20,6 +20,8 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults; public class Configurator { + public static volatile boolean VespaNettyServerCnxnFactory_isSecure = false; + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(Configurator.class.getName()); private static final String ZOOKEEPER_JMX_LOG4J_DISABLE = "zookeeper.jmx.log4j.disable"; static final String ZOOKEEPER_JUTE_MAX_BUFFER = "jute.maxbuffer"; @@ -70,14 +72,14 @@ public class Configurator { // Includes all available commands in 3.5, except 'wchc' and 'wchp' sb.append("4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n"); sb.append("admin.enableServer=false").append("\n"); - // Need NettyServerCnxnFactory to be able to use TLS for communication - sb.append("serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory").append("\n"); + // Use custom connection factory for TLS on client port - see class' Javadoc for rationale + sb.append("serverCnxnFactory=org.apache.zookeeper.server.VespaNettyServerCnxnFactory").append("\n"); sb.append("quorumListenOnAllIPs=true").append("\n"); sb.append("standaloneEnabled=false").append("\n"); sb.append("reconfigEnabled=true").append("\n"); sb.append("skipACL=yes").append("\n"); ensureThisServerIsRepresented(config.myid(), config.server()); - config.server().forEach(server -> addServerToCfg(sb, server)); + config.server().forEach(server -> addServerToCfg(sb, server, config.clientPort())); sb.append(new TlsQuorumConfig().createConfig(config, tlsContext)); sb.append(new TlsClientServerConfig().createConfig(config, tlsContext)); return sb.toString(); @@ -102,7 +104,7 @@ public class Configurator { } } - private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server) { + private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server, int clientPort) { sb.append("server.") .append(server.id()) .append("=") @@ -120,7 +122,9 @@ public class Configurator { sb.append(":") .append("observer"); } - sb.append("\n"); + sb.append(";") + .append(clientPort) + .append("\n"); } static List<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) { @@ -190,12 +194,11 @@ public class Configurator { default: throw new IllegalArgumentException("Unknown value of config setting tlsForClientServerCommunication: " + tlsSetting); } - // ZooKeeper Dynamic Reconfiguration does not support SSL/secure client port - // The secure client port must be configured in the static configuration section instead + sb.append("client.portUnification=").append(portUnification).append("\n"); + // ZooKeeper Dynamic Reconfiguration requires the "non-secure" client port to exist + // This is a hack to override the secure parameter through our connection factory wrapper // https://issues.apache.org/jira/browse/ZOOKEEPER-3577 - sb.append("client.portUnification=").append(portUnification).append("\n") - .append("clientPort=").append(secureClientPort ? 0 : config.clientPort()).append("\n") - .append("secureClientPort=").append(secureClientPort ? config.clientPort() : 0).append("\n"); + VespaNettyServerCnxnFactory_isSecure = secureClientPort; appendSharedTlsConfig(sb, tlsContext); return sb.toString(); diff --git a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java index f0d3df43c4e..47fed6fceac 100644 --- a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java +++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java @@ -171,7 +171,7 @@ public class ConfiguratorTest { "autopurge.snapRetainCount=15\n" + "4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" + "admin.enableServer=false\n" + - "serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory\n" + + "serverCnxnFactory=org.apache.zookeeper.server.VespaNettyServerCnxnFactory\n" + "quorumListenOnAllIPs=true\n" + "standaloneEnabled=false\n" + "reconfigEnabled=true\n" + @@ -181,12 +181,10 @@ public class ConfiguratorTest { private void validateConfigFileSingleHost(File cfgFile) { String expected = commonConfig() + - "server.0=foo:321:123\n" + + "server.0=foo:321:123;2181\n" + "sslQuorum=false\n" + "portUnification=false\n" + - "client.portUnification=false\n" + - "clientPort=2181\n" + - "secureClientPort=0\n"; + "client.portUnification=false\n"; validateConfigFile(cfgFile, expected); } @@ -209,27 +207,23 @@ public class ConfiguratorTest { private void validateConfigFileMultipleHosts(File cfgFile) { String expected = commonConfig() + - "server.0=foo:321:123\n" + - "server.1=bar:432:234\n" + - "server.2=baz:543:345:observer\n" + + "server.0=foo:321:123;2181\n" + + "server.1=bar:432:234;2181\n" + + "server.2=baz:543:345:observer;2181\n" + "sslQuorum=false\n" + "portUnification=false\n" + - "client.portUnification=false\n" + - "clientPort=2181\n" + - "secureClientPort=0\n"; + "client.portUnification=false\n"; validateConfigFile(cfgFile, expected); } private void validateConfigFilePortUnification(File cfgFile) { String expected = commonConfig() + - "server.0=foo:321:123\n" + + "server.0=foo:321:123;2181\n" + "sslQuorum=false\n" + "portUnification=true\n" + tlsQuorumConfig() + "client.portUnification=true\n" + - "clientPort=2181\n" + - "secureClientPort=0\n" + tlsClientServerConfig(); validateConfigFile(cfgFile, expected); } @@ -237,13 +231,11 @@ public class ConfiguratorTest { private void validateConfigFileTlsWithPortUnification(File cfgFile) { String expected = commonConfig() + - "server.0=foo:321:123\n" + + "server.0=foo:321:123;2181\n" + "sslQuorum=true\n" + "portUnification=true\n" + tlsQuorumConfig() + "client.portUnification=true\n" + - "clientPort=2181\n" + - "secureClientPort=0\n" + tlsClientServerConfig(); validateConfigFile(cfgFile, expected); } @@ -251,13 +243,11 @@ public class ConfiguratorTest { private void validateConfigFileTlsOnly(File cfgFile) { String expected = commonConfig() + - "server.0=foo:321:123\n" + + "server.0=foo:321:123;2181\n" + "sslQuorum=true\n" + "portUnification=false\n" + tlsQuorumConfig() + "client.portUnification=false\n" + - "clientPort=0\n" + - "secureClientPort=2181\n" + tlsClientServerConfig(); validateConfigFile(cfgFile, expected); } |