diff options
278 files changed, 4030 insertions, 2031 deletions
diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 0d27ca07f85..a00131e7ba0 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -2395,9 +2395,9 @@ eslint-plugin-import@^2: tsconfig-paths "^3.14.2" eslint-plugin-prettier@^5: - version "5.0.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-5.0.0.tgz#6887780ed95f7708340ec79acfdf60c35b9be57a" - integrity sha512-AgaZCVuYDXHUGxj/ZGu1u8H8CYgDY3iG6w5kUFw4AzMVXzB7VvbKgYR4nATIN+OvUrghMbiDLeimVjVY5ilq3w== + version "5.0.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-5.0.1.tgz#a3b399f04378f79f066379f544e42d6b73f11515" + integrity sha512-m3u5RnR56asrwV/lDC4GHorlW75DsFfmUcjfCYylTUs85dBRnB7VM6xG8eCMJdeDRnppzmxZVf1GEPJvl1JmNg== dependencies: prettier-linter-helpers "^1.0.0" synckit "^0.8.5" @@ -2589,9 +2589,9 @@ execa@^5.0.0: strip-final-newline "^2.0.0" execa@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/execa/-/execa-7.1.1.tgz#3eb3c83d239488e7b409d48e8813b76bb55c9c43" - integrity sha512-wH0eMf/UXckdUYnO21+HDztteVv05rq2GXksxT4fCGeHkBhw1DROXh40wcjMcRqDOWE7iPJ4n3M7e2+YFP+76Q== + version "7.2.0" + resolved "https://registry.yarnpkg.com/execa/-/execa-7.2.0.tgz#657e75ba984f42a70f38928cedc87d6f2d4fe4e9" + integrity sha512-UduyVP7TLB5IcAQl+OzLyLcS/l32W/GLg+AhHJ+ow40FOk2U3SAllPwR44v4vmdFwIWqpdwxxpQbF1n5ta9seA== dependencies: cross-spawn "^7.0.3" get-stream "^6.0.1" @@ -5310,16 +5310,11 @@ tsconfig-paths@^3.14.2: minimist "^1.2.6" strip-bom "^3.0.0" -tslib@^2.0.0, tslib@^2.1.0: +tslib@^2.0.0, tslib@^2.1.0, tslib@^2.5.0, tslib@^2.6.0: version "2.6.2" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== -tslib@^2.5.0, tslib@^2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.0.tgz#b295854684dbda164e181d259a22cd779dcd7bc3" - integrity sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA== - type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 652ba4fda00..e4b1d2edfa4 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -119,7 +119,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"jonmv"}) default boolean useReconfigurableDispatcher() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default int contentLayerMetadataFeatureLevel() { return 0; } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean dynamicHeapSize() { return false; } - @ModelFeatureFlag(owners = {"hmusum"}) default String unknownConfigDefinition() { return "log"; } + @ModelFeatureFlag(owners = {"hmusum"}) default String unknownConfigDefinition() { return "warn"; } @ModelFeatureFlag(owners = {"hmusum"}) default int searchHandlerThreadpool() { return 2; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 7b34c16b8a2..9821f3b9568 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -17,6 +17,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.TenantName; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.config.ApplicationBundlesConfig; import com.yahoo.container.handler.metrics.MetricsProxyApiConfig; @@ -78,6 +79,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH); private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_2 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH + "/*"); + private static final TenantName HOSTED_VESPA = TenantName.from("hosted-vespa"); + public static final int defaultHeapSizePercentageOfAvailableMemory = 85; public static final int heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster = 24; @@ -223,8 +226,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat /** Create list of endpoints, these will be consumed later by LbServicesProducer */ private void createEndpoints(DeployState deployState) { - if (!deployState.isHosted()) return; - if (deployState.getProperties().applicationId().instance().isTester()) return; + if (!configureEndpoints(deployState)) return; // Add endpoints provided by the controller List<String> hosts = getContainers().stream().map(AbstractService::getHostName).sorted().toList(); List<ApplicationClusterEndpoint> endpoints = new ArrayList<>(); @@ -241,6 +243,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat .authMethod(ce.authMethod()) .build()) )); + if (endpoints.stream().noneMatch(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.zone)) { + throw new IllegalArgumentException("Expected at least one " + ApplicationClusterEndpoint.Scope.zone + + " endpoint for cluster '" + name() + "' in application '" + + deployState.getProperties().applicationId() + + "', got " + deployState.getEndpoints()); + } this.endpoints = Collections.unmodifiableList(endpoints); } @@ -374,6 +382,14 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public OnnxModelCost.Calculator onnxModelCost() { return onnxModelCost; } + /** Returns whether the deployment in given deploy state should have endpoints */ + private static boolean configureEndpoints(DeployState deployState) { + if (!deployState.isHosted()) return false; + if (deployState.getProperties().applicationId().instance().isTester()) return false; + if (deployState.getProperties().applicationId().tenant().equals(HOSTED_VESPA)) return false; + return true; + } + public static class MbusParams { // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) final Double maxConcurrentFactor; diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 0e6fdeccfcd..de7e8788f62 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -5,6 +5,8 @@ import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.cloud.config.log.LogdConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -43,6 +45,7 @@ import org.junit.jupiter.api.Test; import java.io.StringReader; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -56,8 +59,8 @@ import static com.yahoo.config.provision.NodeResources.Architecture; import static com.yahoo.config.provision.NodeResources.DiskSpeed; import static com.yahoo.config.provision.NodeResources.StorageType; import static com.yahoo.vespa.defaults.Defaults.getDefaults; -import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; import static com.yahoo.vespa.model.Host.memoryOverheadGb; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; import static com.yahoo.vespa.model.test.utils.ApplicationPackageUtils.generateSchemas; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -183,7 +186,7 @@ public class ModelProvisioningTest { int numberOfHosts = 5; tester.addHosts(numberOfHosts); int numberOfContentNodes = 2; - VespaModel model = tester.createModel(xmlWithNodes, true); + VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("bar.indexing")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); Map<String, ContentCluster> contentClusters = model.getContentClusters(); ContentCluster cluster = contentClusters.get("bar"); @@ -226,7 +229,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(8); tester.addHosts(new NodeResources(20, 200, 2000, 1.0), 1); - VespaModel model = tester.createModel(xmlWithNodes, true); + VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1", "container2")); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(1, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); @@ -249,7 +252,7 @@ public class ModelProvisioningTest { "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(1); - VespaModel model = tester.createModel(xmlWithNodes, true); + VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1")); assertEquals(1, model.hostSystem().getHosts().size()); HostResource host = model.hostSystem().getHosts().iterator().next(); @@ -281,7 +284,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(5); TestLogger logger = new TestLogger(); - VespaModel model = tester.createModel(xmlWithNodes, true, new DeployState.Builder().deployLogger(logger)); + VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1").deployLogger(logger)); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); assertEquals(18, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); @@ -318,7 +321,7 @@ public class ModelProvisioningTest { "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(5); - VespaModel model = tester.createModel(xmlWithNodes, true); + VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1")); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); assertEquals(30, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); @@ -350,7 +353,7 @@ public class ModelProvisioningTest { "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(7); - VespaModel model = tester.createModel(xmlWithNodes, true); + VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1")); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); assertEquals(65, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is normal"); @@ -378,7 +381,7 @@ public class ModelProvisioningTest { "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(5); - VespaModel model = tester.createModel(xmlWithNodes, true); + VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1")); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); @@ -414,7 +417,7 @@ public class ModelProvisioningTest { "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(8); - VespaModel model = tester.createModel(xmlWithNodes, true); + VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1", "container2")); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); @@ -523,7 +526,7 @@ public class ModelProvisioningTest { int numberOfHosts = 67; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("foo")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); // Check container cluster @@ -630,7 +633,7 @@ public class ModelProvisioningTest { int numberOfHosts = 73; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("foo")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); @@ -675,7 +678,7 @@ public class ModelProvisioningTest { int numberOfHosts = 73; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("foo")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); @@ -736,7 +739,7 @@ public class ModelProvisioningTest { int numberOfHosts = 10; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("foo")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); // Check container cluster @@ -784,7 +787,7 @@ public class ModelProvisioningTest { int numberOfHosts = 67; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("foo")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); // Check container cluster @@ -884,7 +887,7 @@ public class ModelProvisioningTest { int numberOfHosts = 21; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("foo")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ClusterControllerContainerCluster clusterControllers = model.getAdmin().getClusterControllers(); @@ -934,7 +937,7 @@ public class ModelProvisioningTest { int numberOfHosts = 11; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "node-1-3-50-09"); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, deployStateWithClusterEndpoints("foo"), "node-1-3-50-09"); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); // Check slobroks clusters @@ -959,7 +962,7 @@ public class ModelProvisioningTest { int numberOfHosts = 12; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "node-1-3-50-03", "node-1-3-50-04"); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, deployStateWithClusterEndpoints("foo"), "node-1-3-50-03", "node-1-3-50-04"); assertEquals(10+2, model.getRoot().hostSystem().getHosts().size()); // Check slobroks clusters @@ -988,7 +991,7 @@ public class ModelProvisioningTest { int numberOfHosts = 16; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true, "node-1-3-50-15", "node-1-3-50-05", "node-1-3-50-04"); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, deployStateWithClusterEndpoints("foo", "bar"), "node-1-3-50-15", "node-1-3-50-05", "node-1-3-50-04"); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); // Check slobroks clusters @@ -1027,7 +1030,7 @@ public class ModelProvisioningTest { int numberOfHosts = 7; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("foo.indexing", "bar.indexing")); assertEquals(7, model.getRoot().hostSystem().getHosts().size()); // Check cluster controllers @@ -1080,7 +1083,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts+1); - VespaModel model = tester.createModel(Zone.defaultZone(), services, true); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, deployStateWithClusterEndpoints("foo")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); Admin admin = model.getAdmin(); @@ -1126,7 +1129,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts+1); - VespaModel model = tester.createModel(Zone.defaultZone(), services, true); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, deployStateWithClusterEndpoints("foo")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); Admin admin = model.getAdmin(); @@ -1179,7 +1182,7 @@ public class ModelProvisioningTest { int numberOfHosts = 6; // We only have 6 content nodes -> 3 groups with redundancy 2 in each VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, false); + VespaModel model = tester.createModel(services, false, deployStateWithClusterEndpoints("bar.indexing")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); @@ -1254,7 +1257,7 @@ public class ModelProvisioningTest { int numberOfHosts = 5; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, false, "node-1-3-50-05", "node-1-3-50-04", "node-1-3-50-03"); + VespaModel model = tester.createModel(Zone.defaultZone(), services, false, deployStateWithClusterEndpoints("bar.indexing"), "node-1-3-50-05", "node-1-3-50-04", "node-1-3-50-03"); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); @@ -1283,7 +1286,9 @@ public class ModelProvisioningTest { int numberOfHosts = 3; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, false, false, true, "node-1-3-50-03"); + VespaModel model = tester.createModel(Zone.defaultZone(), services, false, false, true, + NodeResources.unspecified(), 0, Optional.empty(), + deployStateWithClusterEndpoints("bar.indexing"), "node-1-3-50-03"); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); @@ -1321,7 +1326,7 @@ public class ModelProvisioningTest { int numberOfHosts = 6; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, false); + VespaModel model = tester.createModel(services, false, deployStateWithClusterEndpoints("container")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); @@ -1364,7 +1369,7 @@ public class ModelProvisioningTest { int numberOfHosts = 1; // We only have 1 content node -> 1 groups with redundancy 1 VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, false); + VespaModel model = tester.createModel(services, false, deployStateWithClusterEndpoints("bar.indexing")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); @@ -1443,7 +1448,7 @@ public class ModelProvisioningTest { int numberOfHosts = 5; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, false); + VespaModel model = tester.createModel(services, false, deployStateWithClusterEndpoints("container")); model.hostSystem().getHosts().forEach(host -> assertTrue(host.spec().membership().get().cluster().isExclusive())); } @@ -1469,7 +1474,7 @@ public class ModelProvisioningTest { int numberOfHosts = 1; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, false); + VespaModel model = tester.createModel(services, false, deployStateWithClusterEndpoints("bar.indexing")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); ContentCluster cluster = model.getContentClusters().get("bar"); @@ -1548,7 +1553,9 @@ public class ModelProvisioningTest { tester.addHosts(new NodeResources(8, 200, 1000000, 0.3), 5); // Content-foo tester.addHosts(new NodeResources(10, 64, 200, 0.3), 6); // Content-bar tester.addHosts(new NodeResources(0.5, 2, 10, 0.3), 6); // Cluster-controller - VespaModel model = tester.createModel(services, true, NodeResources.unspecified(), 0); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, false, false, + NodeResources.unspecified(), 0, Optional.empty(), + deployStateWithClusterEndpoints("container", "container2")); assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size()); } @@ -1557,7 +1564,7 @@ public class ModelProvisioningTest { String services = "<?xml version='1.0' encoding='utf-8' ?>" + "<services>" + - " <container version='1.0' id='container'>" + + " <container version='1.0' id='default'>" + " <nodes count='[4, 6]'>" + " <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" + " </nodes>" + @@ -1587,7 +1594,7 @@ public class ModelProvisioningTest { String services = "<?xml version='1.0' encoding='utf-8' ?>" + "<services>" + - " <container version='1.0' id='container'>" + + " <container version='1.0' id='default'>" + " <nodes count='[4, 6]'>" + " <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" + " </nodes>" + @@ -1619,7 +1626,7 @@ public class ModelProvisioningTest { "<services>" + " <admin version='4.0'>" + " </admin>" + - " <container version='1.0' id='container'>" + + " <container version='1.0' id='default'>" + " <nodes count='2'>" + " <resources vcpu='2' memory='8Gb' disk='30Gb'/>" + " </nodes>" + @@ -1678,7 +1685,7 @@ public class ModelProvisioningTest { int numberOfHosts = 3; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("container")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); assertEquals(3, model.getContainerClusters().get("container").getContainers().size()); assertNotNull(model.getAdmin().getLogserver()); @@ -1698,7 +1705,7 @@ public class ModelProvisioningTest { int numberOfHosts = 3; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("container")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); assertEquals("-DfooOption=xyz", model.getContainerClusters().get("container").getContainers().get(0).getAssignedJvmOptions()); } @@ -1739,7 +1746,7 @@ public class ModelProvisioningTest { "</container>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(2); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("foo")); assertEquals(2, model.getHosts().size()); assertEquals(1, model.getContainerClusters().size()); assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); @@ -1797,7 +1804,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.setHosted(true); tester.addHosts(4); - VespaModel model = tester.createModel(new Zone(Environment.dev, RegionName.from("us-central-1")), services, true); + VespaModel model = tester.createModel(new Zone(Environment.dev, RegionName.from("us-central-1")), services, true, deployStateWithClusterEndpoints("foo")); assertEquals(3, model.getHosts().size(), "We get 1 node per cluster and no admin node apart from the dedicated cluster controller"); assertEquals(1, model.getContainerClusters().size()); assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); @@ -1895,7 +1902,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.setHosted(true); tester.addHosts(6); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("container1")); var contentCluster = model.getContentClusters().get("content"); ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); @@ -1924,7 +1931,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.setHosted(true); tester.addHosts(6); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("container1")); var contentCluster = model.getContentClusters().get("content"); ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); @@ -1953,7 +1960,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.setHosted(true); tester.addHosts(6); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("container1")); var contentCluster = model.getContentClusters().get("content"); ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); @@ -2039,7 +2046,7 @@ public class ModelProvisioningTest { "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(6); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("foo")); assertEquals(6, model.getRoot().hostSystem().getHosts().size()); assertEquals(3, model.getAdmin().getSlobroks().size()); assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); @@ -2058,7 +2065,7 @@ public class ModelProvisioningTest { "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(2); - VespaModel model = tester.createModel(services, true); + VespaModel model = tester.createModel(services, true, deployStateWithClusterEndpoints("foo")); assertEquals(2, model.getRoot().hostSystem().getHosts().size()); assertEquals(2, model.getAdmin().getSlobroks().size()); assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); @@ -2342,7 +2349,7 @@ public class ModelProvisioningTest { "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(9); - VespaModel model = tester.createModel(servicesXml, true); + VespaModel model = tester.createModel(servicesXml, true, deployStateWithClusterEndpoints("qrs", "zk")); Map<String, Boolean> tests = Map.of("qrs", false, "zk", true, @@ -2381,7 +2388,7 @@ public class ModelProvisioningTest { "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(9); - VespaModel model = tester.createModel(servicesXml, true, new DeployState.Builder().properties(new TestProperties())); + VespaModel model = tester.createModel(servicesXml, true, deployStateWithClusterEndpoints("qrs").properties(new TestProperties())); var fleetControllerConfigBuilder = new FleetcontrollerConfig.Builder(); model.getConfig(fleetControllerConfigBuilder, "admin/standalone/cluster-controllers/0/components/clustercontroller-content-configurer"); @@ -2400,7 +2407,7 @@ public class ModelProvisioningTest { "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(4); - VespaModel model = tester.createModel(servicesXml, true, "node-1-3-50-04"); + VespaModel model = tester.createModel(Zone.defaultZone(), servicesXml, true, deployStateWithClusterEndpoints("zk"), "node-1-3-50-04"); ApplicationContainerCluster cluster = model.getContainerClusters().get("zk"); assertEquals(1, cluster.getContainers().stream().filter(Container::isRetired).count()); assertEquals(3, cluster.getContainers().stream().filter(c -> !c.isRetired()).count()); @@ -2419,7 +2426,7 @@ public class ModelProvisioningTest { }; VespaModelTester tester = new VespaModelTester(); tester.addHosts(5); - VespaModel model = tester.createModel(servicesXml.apply(3), true); + VespaModel model = tester.createModel(servicesXml.apply(3), true, deployStateWithClusterEndpoints("zk")); { ApplicationContainerCluster cluster = model.getContainerClusters().get("zk"); @@ -2429,7 +2436,7 @@ public class ModelProvisioningTest { assertTrue(config.build().server().stream().noneMatch(ZookeeperServerConfig.Server::joining), "Initial servers are not joining"); } { - VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(3), true, false, false, NodeResources.unspecified(), 0, Optional.of(model), new DeployState.Builder(), "node-1-3-50-04", "node-1-3-50-03"); + VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(3), true, false, false, NodeResources.unspecified(), 0, Optional.of(model), deployStateWithClusterEndpoints("zk"), "node-1-3-50-04", "node-1-3-50-03"); ApplicationContainerCluster cluster = nextModel.getContainerClusters().get("zk"); ZookeeperServerConfig.Builder config = new ZookeeperServerConfig.Builder(); cluster.getContainers().forEach(c -> c.getConfig(config)); @@ -2495,7 +2502,8 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(new NodeResources(1, 3, 10, 5, NodeResources.DiskSpeed.slow), 5); - VespaModel model = tester.createModel(services, true, NodeResources.unspecified(), 0); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, false, false, NodeResources.unspecified(), 0, Optional.empty(), deployStateWithClusterEndpoints("test.indexing")); + ContentSearchCluster cluster = model.getContentClusters().get("test").getSearch(); assertEquals(2, cluster.getSearchNodes().size()); assertEquals(40, getProtonConfig(cluster, 0).hwinfo().disk().writespeed(), 0.001); @@ -2519,7 +2527,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(new NodeResources(1, 3, 10, 5), 5); - VespaModel model = tester.createModel(services, true, new NodeResources(1.0, 3.0, 9.0, 1.0), 0); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, false, false, new NodeResources(1.0, 3.0, 9.0, 1.0), 0, Optional.empty(), deployStateWithClusterEndpoints("test.indexing")); ContentSearchCluster cluster = model.getContentClusters().get("test").getSearch(); assertEquals(2, cluster.getSearchNodes().size()); } @@ -2568,7 +2576,7 @@ public class ModelProvisioningTest { VespaModelTester tester = new VespaModelTester(); tester.addHosts(new NodeResources(1, 3, 10, 1), 4); tester.addHosts(new NodeResources(1, 128, 100, 0.3), 1); - VespaModel model = tester.createModel(services, true, NodeResources.unspecified(), 0); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, false, false, NodeResources.unspecified(), 0, Optional.empty(), deployStateWithClusterEndpoints("test.indexing")); ContentSearchCluster cluster = model.getContentClusters().get("test").getSearch(); ProtonConfig cfg = getProtonConfig(model, cluster.getSearchNodes().get(0).getConfigId()); assertEquals(2000, cfg.flush().memory().maxtlssize()); // from config override @@ -2590,7 +2598,7 @@ public class ModelProvisioningTest { tester.useDedicatedNodeForLogserver(useDedicatedNodeForLogserver); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(Zone.defaultZone(), services, true); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, deployStateWithClusterEndpoints("foo")); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); Admin admin = model.getAdmin(); @@ -2632,6 +2640,15 @@ public class ModelProvisioningTest { return hostSystem.getHosts().stream().map(HostResource::getHost).anyMatch(host -> host.getHostname().equals(hostname)); } + private static DeployState.Builder deployStateWithClusterEndpoints(String... cluster) { + Set<ContainerEndpoint> endpoints = Arrays.stream(cluster) + .map(c -> new ContainerEndpoint(c, + ApplicationClusterEndpoint.Scope.zone, + List.of(c + ".example.com"))) + .collect(Collectors.toSet()); + return new DeployState.Builder().endpoints(endpoints); + } + record TestLogger(List<LogMessage> msgs) implements DeployLogger { public TestLogger() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java index 424d3589e6a..4df9f261dfe 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.model; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.provision.InMemoryProvisioner; @@ -20,14 +22,14 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import java.time.Duration; +import java.util.List; import java.util.Map; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; /** * @author bratseth @@ -251,6 +253,7 @@ public class ClusterInfoTest { .setCloudAccount(account) .setApplicationId(ApplicationId.from(TenantName.defaultName(), ApplicationName.defaultName(), InstanceName.from(instance))) .setZone(new Zone(Environment.prod, RegionName.from(region)))) + .endpoints(Set.of(new ContainerEndpoint("testcontainer", ApplicationClusterEndpoint.Scope.zone, List.of("tc.example.com")))) .modelHostProvisioner(provisioner) .provisioned(provisioner.provisioned()) .build(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index 80d1e6eac91..af825ca544a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -6,8 +6,8 @@ import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; import ai.vespa.metricsproxy.service.VespaServicesConfig; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.test.VespaModelTester; import org.junit.jupiter.api.Test; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; @@ -16,11 +16,12 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.T import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.containerConfigId; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; - import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getRpcConnectorConfig; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getVespaServicesConfig; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author gjoranv @@ -29,11 +30,8 @@ public class MetricsProxyContainerTest { @Test void one_metrics_proxy_container_is_added_to_every_node() { - var numberOfHosts = 7; - var tester = new VespaModelTester(); - tester.addHosts(numberOfHosts); - - VespaModel model = tester.createModel(hostedServicesWithManyNodes(), true); + int numberOfHosts = 7; + VespaModel model = getModel(hostedServicesWithManyNodes(), hosted, new DeployState.Builder(), numberOfHosts); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); for (var host : model.hostSystem().getHosts()) { @@ -48,11 +46,8 @@ public class MetricsProxyContainerTest { @Test void one_metrics_proxy_container_is_added_to_every_node_also_when_dedicated_CCC() { - var numberOfHosts = 7; - var tester = new VespaModelTester(); - tester.addHosts(numberOfHosts); - - VespaModel model = tester.createModel(hostedServicesWithManyNodes(), true); + int numberOfHosts = 7; + VespaModel model = getModel(hostedServicesWithManyNodes(), hosted, new DeployState.Builder(), numberOfHosts); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); for (var host : model.hostSystem().getHosts()) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index be8b785faf9..332426ff9a8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -9,12 +9,17 @@ import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; import ai.vespa.metricsproxy.service.VespaServicesConfig; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.test.VespaModelTester; +import java.util.List; +import java.util.Set; + import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; @@ -42,11 +47,18 @@ class MetricsProxyModelTester { } static VespaModel getModel(String servicesXml, TestMode testMode, DeployState.Builder builder) { - var numberOfHosts = testMode == hosted ? 4 : 1; + return getModel(servicesXml, testMode, new DeployState.Builder(), 4); + } + + static VespaModel getModel(String servicesXml, TestMode testMode, DeployState.Builder builder, int hostCount) { + var numberOfHosts = testMode == hosted ? hostCount : 1; var tester = new VespaModelTester(); tester.addHosts(numberOfHosts); tester.setHosted(testMode == hosted); - if (testMode == hosted) tester.setApplicationId(MY_TENANT, MY_APPLICATION, MY_INSTANCE); + if (testMode == hosted) { + tester.setApplicationId(MY_TENANT, MY_APPLICATION, MY_INSTANCE); + builder.endpoints(Set.of(new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.zone, List.of("foo.example.com")))); + } return tester.createModel(servicesXml, true, builder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidatorTest.java index 6695c4ce2d8..25053c536da 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlFilterExcludeValidatorTest.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.MapConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; @@ -20,9 +22,11 @@ import org.junit.jupiter.api.Test; import org.xml.sax.SAXException; import java.io.IOException; +import java.util.List; +import java.util.Set; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -98,6 +102,7 @@ public class AccessControlFilterExcludeValidatorTest { .setHostedVespa(true) .setAthenzDomain(AthenzDomain.from("foo.bar")) .allowDisableMtls(allowExcludes)) + .endpoints(Set.of(new ContainerEndpoint("container-cluster-with-access-control", ApplicationClusterEndpoint.Scope.zone, List.of("example.com")))) .deployLogger(logger) .zone(zone) .build(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidatorTest.java index 2c68873353e..8acbf00a5a3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudDataPlaneFilterValidatorTest.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -35,6 +37,7 @@ import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -149,6 +152,7 @@ public class CloudDataPlaneFilterValidatorTest { new TestProperties() .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY"))) .setHostedVespa(true)) + .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("c.example.com")))) .zone(new Zone(SystemName.PublicCd, Environment.dev, RegionName.defaultName())) .build(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidatorTest.java index 655703b2159..58aa0e8625e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudHttpConnectorValidatorTest.java @@ -3,12 +3,17 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.vespa.model.VespaModel; import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Set; + import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -96,6 +101,7 @@ class CloudHttpConnectorValidatorTest { .withServices(servicesXml) .build()) .properties(new TestProperties().setHostedVespa(hosted)) + .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("c.example.com")))) .build(); var model = new VespaModel(new NullConfigModelRegistry(), state); new CloudHttpConnectorValidator().validate(model, state); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidatorTest.java index 99eeb0882af..2aa678fd34b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudUserFilterValidatorTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; @@ -12,6 +14,8 @@ import org.junit.jupiter.api.Test; import org.xml.sax.SAXException; import java.io.IOException; +import java.util.List; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -59,6 +63,7 @@ class CloudUserFilterValidatorTest { .build(); DeployState deployState = new DeployState.Builder() .applicationPackage(app) + .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("container.example.com")))) .properties(new TestProperties().setHostedVespa(isHosted).setAllowUserFilters(false)) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java index 6d605e8b964..43c51bea04a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ContainerInCloudValidatorTest.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; @@ -11,8 +13,9 @@ import org.junit.jupiter.api.Test; import org.xml.sax.SAXException; import java.io.IOException; +import java.util.List; +import java.util.Set; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -50,10 +53,13 @@ public class ContainerInCloudValidatorTest { ApplicationPackage app = new MockApplicationPackage.Builder() .withServices(servicesXml) .build(); - DeployState deployState = new DeployState.Builder() + DeployState.Builder builder = new DeployState.Builder() .applicationPackage(app) - .properties(new TestProperties().setHostedVespa(isHosted).setAllowUserFilters(false)) - .build(); + .properties(new TestProperties().setHostedVespa(isHosted).setAllowUserFilters(false)); + if (isHosted) { + builder.endpoints(Set.of(new ContainerEndpoint("routing", ApplicationClusterEndpoint.Scope.zone, List.of("routing.example.com")))); + } + DeployState deployState = builder.build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); new ContainerInCloudValidator().validate(model, deployState); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java index de5f8c35c01..821ad1be8fa 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -14,7 +16,9 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.Optional; +import java.util.Set; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -73,6 +77,7 @@ public class EndpointCertificateSecretsValidatorTest { DeployState.Builder builder = new DeployState.Builder() .applicationPackage(app) .zone(new Zone(Environment.prod, RegionName.from("foo"))) + .endpoints(Set.of(new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .properties( new TestProperties() .setHostedVespa(true) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index 06b771bd3ee..d6da03f5b94 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -7,6 +7,8 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -19,6 +21,8 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.net.URI; +import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -85,6 +89,7 @@ class JvmHeapSizeValidatorTest { .withServices(servicesXml) .build()) .modelHostProvisioner(new InMemoryProvisioner(5, new NodeResources(4, nodeGb, 125, 0.3), true)) + .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("c.example.com")))) .properties(new TestProperties().setHostedVespa(true).setDynamicHeapSize(true)) .onnxModelCost(new ModelCostDummy(modelCostBytes)) .build(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 7a6ca0fe2f0..89f81dfdaef 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.fail; */ public class QuotaValidatorTest { + private static final String CONTAINER_CLUSTER = "testCluster.indexing"; private final Zone publicZone = new Zone(SystemName.Public, Environment.prod, RegionName.from("foo")); private final Zone publicCdZone = new Zone(SystemName.PublicCd, Environment.prod, RegionName.from("foo")); private final Zone devZone = new Zone(SystemName.Public, Environment.dev, RegionName.from("foo")); @@ -27,14 +28,14 @@ public class QuotaValidatorTest { @Test void test_deploy_under_quota() { var tester = new ValidationTester(8, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); - tester.deploy(null, getServices(4), Environment.prod, null); + tester.deploy(null, getServices(4), Environment.prod, null, CONTAINER_CLUSTER); } @Test void test_deploy_above_quota_clustersize() { var tester = new ValidationTester(14, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); try { - tester.deploy(null, getServices(11), Environment.prod, null); + tester.deploy(null, getServices(11), Environment.prod, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { assertEquals("Clusters testCluster exceeded max cluster size of 10", e.getMessage()); @@ -45,7 +46,7 @@ public class QuotaValidatorTest { void test_deploy_above_quota_budget() { var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); try { - tester.deploy(null, getServices(10), Environment.prod, null); + tester.deploy(null, getServices(10), Environment.prod, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { assertEquals("The resources used cost $1.63 but your quota is $1.25: Contact support to upgrade your plan.", e.getMessage()); @@ -56,7 +57,7 @@ public class QuotaValidatorTest { void test_deploy_above_quota_budget_in_publiccd() { var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota.withBudget(BigDecimal.ONE)).setZone(publicCdZone)); try { - tester.deploy(null, getServices(10), Environment.prod, null); + tester.deploy(null, getServices(10), Environment.prod, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { assertEquals("publiccd: The resources used cost $1.63 but your quota is $1.00: Contact support to upgrade your plan.", e.getMessage()); @@ -67,7 +68,7 @@ public class QuotaValidatorTest { void test_deploy_max_resources_above_quota() { var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone)); try { - tester.deploy(null, getServices(10), Environment.prod, null); + tester.deploy(null, getServices(10), Environment.prod, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { assertEquals("publiccd: The resources used cost $1.63 but your quota is $1.25: Contact support to upgrade your plan.", e.getMessage()); @@ -82,7 +83,7 @@ public class QuotaValidatorTest { // There is downscaling to 1 node per cluster in dev try { - tester.deploy(null, getServices(2, false), Environment.dev, null); + tester.deploy(null, getServices(2, false), Environment.dev, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { assertEquals("The resources used cost $0.16 but your quota is $0.01: Contact support to upgrade your plan.", e.getMessage()); @@ -90,7 +91,7 @@ public class QuotaValidatorTest { // Override so that we will get 2 nodes in content cluster try { - tester.deploy(null, getServices(2, true), Environment.dev, null); + tester.deploy(null, getServices(2, true), Environment.dev, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { assertEquals("The resources used cost $0.33 but your quota is $0.01: Contact support to upgrade your plan.", e.getMessage()); @@ -102,7 +103,7 @@ public class QuotaValidatorTest { var quota = Quota.unlimited().withBudget(BigDecimal.valueOf(-1)); var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); try { - tester.deploy(null, getServices(10), Environment.prod, null); + tester.deploy(null, getServices(10), Environment.prod, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { assertEquals("The resources used cost $-.-- but your quota is $--.--: Please free up some capacity.", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java index e551c7f04e8..ae23b3b722d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; @@ -12,6 +14,9 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Set; + import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -80,6 +85,7 @@ public class SecretStoreValidatorTest { DeployState.Builder builder = new DeployState.Builder() .applicationPackage(app) .zone(new Zone(Environment.prod, RegionName.from("foo"))) + .endpoints(Set.of(new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .properties(new TestProperties().setHostedVespa(true)); final DeployState deployState = builder.build(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java index 53f01cd6356..9a2f9fadac6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; @@ -16,8 +18,11 @@ import org.junit.jupiter.api.Test; import org.xml.sax.SAXException; import java.io.IOException; +import java.util.List; +import java.util.Set; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author bjorncs @@ -99,6 +104,7 @@ public class UriBindingsValidatorTest { .deployLogger(deployLogger) .zone(testProperties.zone()) .properties(testProperties) + .endpoints(Set.of(new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); new UriBindingsValidator().validate(model, deployState); @@ -120,7 +126,7 @@ public class UriBindingsValidatorTest { return String.join( "\n", "<services version='1.0'>", - " <container version='1.0'>", + " <container version='1.0' id='default'>", " <http>", " <server port='8080' id='main' />", " <filtering>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java index 23edd3e2f87..837de946e36 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -20,7 +22,9 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Set; import static com.yahoo.config.provision.Environment.prod; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -85,6 +89,7 @@ public class UrlConfigValidatorTest { var builder = new DeployState.Builder() .applicationPackage(app) .zone(new Zone(systemName, prod, RegionName.from("us-east-3"))) + .endpoints(Set.of(new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .properties(new TestProperties().setHostedVespa(isHosted)) .fileRegistry(new MockFileRegistry()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java index 06505171210..8dc07d8857d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.collections.Pair; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -20,7 +22,11 @@ import java.time.Instant; import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.Arrays; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.config.model.test.MockApplicationPackage.BOOK_SCHEMA; import static com.yahoo.config.model.test.MockApplicationPackage.MUSIC_SCHEMA; @@ -67,12 +73,14 @@ public class ValidationTester { * @param services the services file content * @param environment the environment this deploys to * @param validationOverrides the validation overrides file content, or null if none + * @param containerCluster container cluster(s) which are declared in services * @return the new model and any change actions */ public Pair<VespaModel, List<ConfigChangeAction>> deploy(VespaModel previousModel, String services, Environment environment, - String validationOverrides) { + String validationOverrides, + String... containerCluster) { Instant now = LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant(); Provisioned provisioned = hostProvisioner.startProvisionedRecording(); ApplicationPackage newApp = new MockApplicationPackage.Builder() @@ -81,10 +89,16 @@ public class ValidationTester { .withValidationOverrides(validationOverrides) .build(); VespaModelCreatorWithMockPkg newModelCreator = new VespaModelCreatorWithMockPkg(newApp); + Stream<String> clusters = containerCluster.length == 0 ? Stream.of("default") : Arrays.stream(containerCluster); + Set<ContainerEndpoint> containerEndpoints = clusters.map(name -> new ContainerEndpoint(name, + ApplicationClusterEndpoint.Scope.zone, + List.of(name + ".example.com"))) + .collect(Collectors.toSet()); DeployState.Builder deployStateBuilder = new DeployState.Builder() .zone(new Zone(SystemName.defaultSystem(), environment, RegionName.defaultName())) + .endpoints(containerEndpoints) .applicationPackage(newApp) .properties(properties) .modelHostProvisioner(hostProvisioner) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java index bafd41f4d76..362e74bb2e9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java @@ -1,18 +1,14 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; -import com.yahoo.collections.Pair; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.ValidationTester; import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; -import java.util.List; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -25,9 +21,9 @@ public class ContentClusterRemovalValidatorTest { @Test void testContentRemovalValidation() { - VespaModel previous = tester.deploy(null, getServices("contentClusterId"), Environment.prod, null).getFirst(); + VespaModel previous = tester.deploy(null, getServices("contentClusterId"), Environment.prod, null, "contentClusterId.indexing").getFirst(); try { - tester.deploy(previous, getServices("newContentClusterId"), Environment.prod, null); + tester.deploy(previous, getServices("newContentClusterId"), Environment.prod, null, "newContentClusterId.indexing"); fail("Expected exception due to content cluster id change"); } catch (IllegalArgumentException expected) { @@ -39,8 +35,8 @@ public class ContentClusterRemovalValidatorTest { @Test void testOverridingContentRemovalValidation() { - VespaModel previous = tester.deploy(null, getServices("contentClusterId"), Environment.prod, null).getFirst(); - var result = tester.deploy(previous, getServices("newContentClusterId"), Environment.prod, removalOverride); // Allowed due to override + VespaModel previous = tester.deploy(null, getServices("contentClusterId"), Environment.prod, null, "contentClusterId.indexing").getFirst(); + var result = tester.deploy(previous, getServices("newContentClusterId"), Environment.prod, removalOverride, "newContentClusterId.indexing"); // Allowed due to override assertEquals(result.getFirst().getContainerClusters().values().stream() .flatMap(cluster -> cluster.getContainers().stream()) .map(container -> container.getServiceInfo()) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidatorTest.java index e3a20a22141..7b374870ed2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentTypeRemovalValidatorTest.java @@ -24,9 +24,9 @@ public class ContentTypeRemovalValidatorTest { void testContentTypeRemovalValidation() { ValidationTester tester = new ValidationTester(); - VespaModel previous = tester.deploy(null, getServices("music"), Environment.prod, null).getFirst(); + VespaModel previous = tester.deploy(null, getServices("music"), Environment.prod, null, "test.indexing").getFirst(); try { - tester.deploy(previous, getServices("book"), Environment.prod, null); + tester.deploy(previous, getServices("book"), Environment.prod, null, "test.indexing"); fail("Expected exception due to removal of schema 'music"); } catch (IllegalArgumentException expected) { @@ -41,8 +41,8 @@ public class ContentTypeRemovalValidatorTest { void testOverridingContentTypeRemovalValidation() { ValidationTester tester = new ValidationTester(); - VespaModel previous = tester.deploy(null, getServices("music"), Environment.prod, null).getFirst(); - tester.deploy(previous, getServices("book"), Environment.prod, removalOverride); // Allowed due to override + VespaModel previous = tester.deploy(null, getServices("music"), Environment.prod, null, "test.indexing").getFirst(); + tester.deploy(previous, getServices("book"), Environment.prod, removalOverride, "test.indexing"); // Allowed due to override } private static String getServices(String documentType) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidatorTest.java index 2131ecb4879..cdc80754194 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidatorTest.java @@ -25,9 +25,9 @@ public class GlobalDocumentChangeValidatorTest { private void testChangeGlobalAttribute(boolean allowed, boolean oldGlobal, boolean newGlobal, String validationOverrides) { ValidationTester tester = new ValidationTester(); - VespaModel oldModel = tester.deploy(null, getServices(oldGlobal), Environment.prod, validationOverrides).getFirst(); + VespaModel oldModel = tester.deploy(null, getServices(oldGlobal), Environment.prod, validationOverrides, "default.indexing").getFirst(); try { - tester.deploy(oldModel, getServices(newGlobal), Environment.prod, validationOverrides).getSecond(); + tester.deploy(oldModel, getServices(newGlobal), Environment.prod, validationOverrides, "default.indexing").getSecond(); assertTrue(allowed); } catch (IllegalStateException e) { assertFalse(allowed); @@ -37,7 +37,8 @@ public class GlobalDocumentChangeValidatorTest { e.getMessage()); } } - private static final String getServices(boolean isGlobal) { + + private static String getServices(boolean isGlobal) { return "<services version='1.0'>" + " <content id='default' version='1.0'>" + " <redundancy>1</redundancy>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java index 3a3a1aff8af..3fd3180b37e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java @@ -10,9 +10,10 @@ import com.yahoo.vespa.model.application.validation.ValidationTester; import org.junit.jupiter.api.Test; import java.util.List; -import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * @author bratseth @@ -33,7 +34,7 @@ public class IndexingModeChangeValidatorTest { } catch (ValidationException e) { assertEquals("indexing-mode-change:\n" + - "\tDocument type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'streaming'\n" + + "\tDocument type 'music' in cluster 'default-content' changed indexing mode from 'indexed' to 'streaming'\n" + "To allow this add <allow until='yyyy-mm-dd'>indexing-mode-change</allow> to validation-overrides.xml, see https://docs.vespa.ai/en/reference/validation-overrides.html", e.getMessage()); } @@ -49,7 +50,7 @@ public class IndexingModeChangeValidatorTest { tester.deploy(oldModel, getServices("streaming"), Environment.prod, validationOverrides).getSecond(); assertReindexingChange( // allowed=true due to validation override - "Document type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'streaming'", + "Document type 'music' in cluster 'default-content' changed indexing mode from 'indexed' to 'streaming'", changeActions); } @@ -63,7 +64,7 @@ public class IndexingModeChangeValidatorTest { tester.deploy(oldModel, getServices("store-only"), Environment.prod, validationOverrides).getSecond(); assertReindexingChange( // allowed=true due to validation override - "Document type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'store-only'", + "Document type 'music' in cluster 'default-content' changed indexing mode from 'indexed' to 'store-only'", changeActions); } @@ -79,10 +80,10 @@ public class IndexingModeChangeValidatorTest { private static String getServices(String indexingMode) { return "<services version='1.0'>" + - " <container id='default-container' version='1.0'>" + + " <container id='default' version='1.0'>" + " <nodes count='1'/>" + " </container>" + - " <content id='default' version='1.0'>" + + " <content id='default-content' version='1.0'>" + " <redundancy>1</redundancy>" + " <documents>" + " <document type='music' mode='" + indexingMode + "'/>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java index 1f4e9029d83..afa36ac271e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java @@ -1,7 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -17,8 +19,10 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; /** * @author bratseth @@ -50,7 +54,7 @@ public class NodeResourceChangeValidatorTest { ConfigChangeAction contentAction = validate(model(1, 1, 1, 1), model(1, 1, 2, 1)).get(0); assertEquals(ConfigChangeAction.Type.RESTART, contentAction.getType()); assertEquals("service 'searchnode' of type searchnode on host3", contentAction.getServices().get(0).toString()); - assertEquals(false, contentAction.ignoreForInternalRedeploy()); + assertFalse(contentAction.ignoreForInternalRedeploy()); } private List<ConfigChangeAction> validate(VespaModel current, VespaModel next) { @@ -61,6 +65,10 @@ public class NodeResourceChangeValidatorTest { var properties = new TestProperties(); properties.setHostedVespa(true); var deployState = new DeployState.Builder().properties(properties) + .endpoints(Set.of( + new ContainerEndpoint("container1", ApplicationClusterEndpoint.Scope.zone, List.of("c1.example.com")), + new ContainerEndpoint("container2", ApplicationClusterEndpoint.Scope.zone, List.of("c2.example.com")) + )) .modelHostProvisioner(new Provisioner()); return new VespaModelCreatorWithMockPkg( null, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RedundancyChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RedundancyChangeValidatorTest.java index c2cceb95c25..e2cd48cc72b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RedundancyChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RedundancyChangeValidatorTest.java @@ -24,8 +24,8 @@ public class RedundancyChangeValidatorTest { public void testChangingRedundancyToOne() { try { var tester = new ValidationTester(6); - VespaModel previous = tester.deploy(null, getServices("test", 2), Environment.prod, null).getFirst(); - tester.deploy(previous, getServices("test", 1), Environment.prod, null); + VespaModel previous = tester.deploy(null, getServices("test", 2), Environment.prod, null, "test.indexing").getFirst(); + tester.deploy(previous, getServices("test", 1), Environment.prod, null, "test.indexing"); fail("Expected exception"); } catch (IllegalArgumentException e) { @@ -41,11 +41,11 @@ public class RedundancyChangeValidatorTest { @Test public void testChangingRedundancyToOneWithValidationOverride() { var tester = new ValidationTester(6); - VespaModel previous = tester.deploy(null, getServices("test", 2), Environment.prod, null).getFirst(); - previous = tester.deploy(previous, getServices("test", 1), Environment.prod, redundancyOverride).getFirst(); + VespaModel previous = tester.deploy(null, getServices("test", 2), Environment.prod, null, "test.indexing").getFirst(); + previous = tester.deploy(previous, getServices("test", 1), Environment.prod, redundancyOverride, "test.indexing").getFirst(); // Staying at one does not require an override - tester.deploy(previous, getServices("test", 1), Environment.prod, null); + tester.deploy(previous, getServices("test", 1), Environment.prod, null, "test.indexing"); } private static String getServices(String contentClusterId, int redundancy) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidatorTest.java index 6a1315db318..c2421e32a7f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidatorTest.java @@ -21,9 +21,9 @@ public class RedundancyIncreaseValidatorTest { @Test void testRedundancyIncreaseValidation() { - VespaModel previous = tester.deploy(null, getServices(2), Environment.prod, null).getFirst(); + VespaModel previous = tester.deploy(null, getServices(2), Environment.prod, null, "contentClusterId.indexing").getFirst(); try { - tester.deploy(previous, getServices(3), Environment.prod, null); + tester.deploy(previous, getServices(3), Environment.prod, null, "contentClusterId.indexing"); fail("Expected exception due to redundancy increase"); } catch (IllegalArgumentException expected) { @@ -37,8 +37,8 @@ public class RedundancyIncreaseValidatorTest { @Test void testOverridingContentRemovalValidation() { - VespaModel previous = tester.deploy(null, getServices(2), Environment.prod, null).getFirst(); - tester.deploy(previous, getServices(3), Environment.prod, redundancyIncreaseOverride); // Allowed due to override + VespaModel previous = tester.deploy(null, getServices(2), Environment.prod, null, "contentClusterId.indexing").getFirst(); + tester.deploy(previous, getServices(3), Environment.prod, redundancyIncreaseOverride, "contentClusterId.indexing"); // Allowed due to override } private static String getServices(int redundancy) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java index e90e5ce3221..f9c0d00ac2c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java @@ -20,6 +20,8 @@ import static org.junit.jupiter.api.Assertions.fail; */ public class ResourcesReductionValidatorTest { + private static final String CONTAINER_CLUSTER = "default.indexing"; + private final NodeResources hostResources = new NodeResources(64, 128, 1000, 10); private final InMemoryProvisioner provisioner = new InMemoryProvisioner(30, hostResources, true, InMemoryProvisioner.defaultHostResources); private final InMemoryProvisioner provisionerSelfHosted = new InMemoryProvisioner(30, hostResources, true, NodeResources.unspecified()); @@ -30,9 +32,9 @@ public class ResourcesReductionValidatorTest { void fail_when_reduction_by_over_50_percent() { var fromResources = new NodeResources(8, 64, 800, 1); var toResources = new NodeResources(8, 16, 800, 1); - VespaModel previous = tester.deploy(null, contentServices(6, fromResources), Environment.prod, null).getFirst(); + VespaModel previous = tester.deploy(null, contentServices(6, fromResources), Environment.prod, null, CONTAINER_CLUSTER).getFirst(); try { - tester.deploy(previous, contentServices(6, toResources), Environment.prod, null); + tester.deploy(previous, contentServices(6, toResources), Environment.prod, null, CONTAINER_CLUSTER); fail("Expected exception due to resources reduction"); } catch (IllegalArgumentException expected) { assertResourceReductionException(expected, @@ -45,9 +47,9 @@ public class ResourcesReductionValidatorTest { void fail_when_reducing_multiple_resources_by_over_50_percent() { var fromResources = new NodeResources(8, 64, 800, 1); var toResources = new NodeResources(3, 16, 200, 1); - VespaModel previous = tester.deploy(null, contentServices(6, fromResources), Environment.prod, null).getFirst(); + VespaModel previous = tester.deploy(null, contentServices(6, fromResources), Environment.prod, null, CONTAINER_CLUSTER).getFirst(); try { - tester.deploy(previous, contentServices(6, toResources), Environment.prod, null); + tester.deploy(previous, contentServices(6, toResources), Environment.prod, null, CONTAINER_CLUSTER); fail("Expected exception due to resources reduction"); } catch (IllegalArgumentException expected) { assertResourceReductionException(expected, @@ -58,20 +60,20 @@ public class ResourcesReductionValidatorTest { @Test void small_resource_decrease_is_allowed() { - VespaModel previous = tester.deploy(null, contentServices(6, new NodeResources(1.5, 64, 800, 1)), Environment.prod, null).getFirst(); - tester.deploy(previous, contentServices(6, new NodeResources(.5, 48, 600, 1)), Environment.prod, null); + VespaModel previous = tester.deploy(null, contentServices(6, new NodeResources(1.5, 64, 800, 1)), Environment.prod, null, CONTAINER_CLUSTER).getFirst(); + tester.deploy(previous, contentServices(6, new NodeResources(.5, 48, 600, 1)), Environment.prod, null, CONTAINER_CLUSTER); } @Test void reorganizing_resources_is_allowed() { - VespaModel previous = tester.deploy(null, contentServices(12, new NodeResources(2, 10, 100, 1)), Environment.prod, null).getFirst(); - tester.deploy(previous, contentServices(4, new NodeResources(6, 30, 300, 1)), Environment.prod, null); + VespaModel previous = tester.deploy(null, contentServices(12, new NodeResources(2, 10, 100, 1)), Environment.prod, null, CONTAINER_CLUSTER).getFirst(); + tester.deploy(previous, contentServices(4, new NodeResources(6, 30, 300, 1)), Environment.prod, null, CONTAINER_CLUSTER); } @Test void overriding_resource_decrease() { - VespaModel previous = tester.deploy(null, contentServices(6, new NodeResources(8, 64, 800, 1)), Environment.prod, null).getFirst(); - tester.deploy(previous, contentServices(6, new NodeResources(8, 16, 800, 1)), Environment.prod, resourcesReductionOverride); // Allowed due to override + VespaModel previous = tester.deploy(null, contentServices(6, new NodeResources(8, 64, 800, 1)), Environment.prod, null, CONTAINER_CLUSTER).getFirst(); + tester.deploy(previous, contentServices(6, new NodeResources(8, 16, 800, 1)), Environment.prod, resourcesReductionOverride, CONTAINER_CLUSTER); // Allowed due to override } @Test @@ -108,23 +110,22 @@ public class ResourcesReductionValidatorTest { void reduction_is_detected_when_going_from_unspecified_resources_content() { NodeResources toResources = defaultResources.withDiskGb(defaultResources.diskGb() / 5); try { - VespaModel previous = tester.deploy(null, contentServices(6, null), Environment.prod, null).getFirst(); - tester.deploy(previous, contentServices(6, toResources), Environment.prod, null); + VespaModel previous = tester.deploy(null, contentServices(6, null), Environment.prod, null, CONTAINER_CLUSTER).getFirst(); + tester.deploy(previous, contentServices(6, toResources), Environment.prod, null, CONTAINER_CLUSTER); fail("Expected exception due to resources reduction"); + } catch (IllegalArgumentException expected) { + assertResourceReductionException(expected, + defaultResources.multipliedBy(6), + toResources.multipliedBy(6)); } - catch (IllegalArgumentException expected) { - assertResourceReductionException(expected, - defaultResources.multipliedBy(6), - toResources.multipliedBy(6)); - } } @Test void reduction_is_detected_when_going_to_unspecified_resources_content() { NodeResources fromResources = defaultResources.withVcpu(defaultResources.vcpu() * 3); try { - VespaModel previous = tester.deploy(null, contentServices(6, fromResources), Environment.prod, null).getFirst(); - tester.deploy(previous, contentServices(6, null), Environment.prod, null); + VespaModel previous = tester.deploy(null, contentServices(6, fromResources), Environment.prod, null, CONTAINER_CLUSTER).getFirst(); + tester.deploy(previous, contentServices(6, null), Environment.prod, null, CONTAINER_CLUSTER); fail("Expected exception due to resources reduction"); } catch (IllegalArgumentException expected) { @@ -140,8 +141,8 @@ public class ResourcesReductionValidatorTest { int toNodes = 14; try { ValidationTester tester = new ValidationTester(33); - VespaModel previous = tester.deploy(null, contentServices(fromNodes, null), Environment.prod, null).getFirst(); - tester.deploy(previous, contentServices(toNodes, null), Environment.prod, null); + VespaModel previous = tester.deploy(null, contentServices(fromNodes, null), Environment.prod, null, CONTAINER_CLUSTER).getFirst(); + tester.deploy(previous, contentServices(toNodes, null), Environment.prod, null, CONTAINER_CLUSTER); fail("Expected exception due to resources reduction"); } catch (IllegalArgumentException expected) { @@ -156,9 +157,9 @@ public class ResourcesReductionValidatorTest { void testSizeReductionValidationSelfhosted() { var tester = new ValidationTester(provisionerSelfHosted); - VespaModel previous = tester.deploy(null, contentServices(10, null), Environment.prod, null).getFirst(); + VespaModel previous = tester.deploy(null, contentServices(10, null), Environment.prod, null, CONTAINER_CLUSTER).getFirst(); try { - tester.deploy(previous, contentServices(4, null), Environment.prod, null); + tester.deploy(previous, contentServices(4, null), Environment.prod, null, CONTAINER_CLUSTER); fail("Expected exception due to resources reduction"); } catch (IllegalArgumentException expected) { @@ -174,16 +175,16 @@ public class ResourcesReductionValidatorTest { void testSizeReductionValidationMinimalDecreaseIsAllowed() { ValidationTester tester = new ValidationTester(30); - VespaModel previous = tester.deploy(null, contentServices(3, null), Environment.prod, null).getFirst(); - tester.deploy(previous, contentServices(2, null), Environment.prod, null); + VespaModel previous = tester.deploy(null, contentServices(3, null), Environment.prod, null, CONTAINER_CLUSTER).getFirst(); + tester.deploy(previous, contentServices(2, null), Environment.prod, null, CONTAINER_CLUSTER); } @Test void testOverridingSizeReductionValidation() { ValidationTester tester = new ValidationTester(33); - VespaModel previous = tester.deploy(null, contentServices(30, null), Environment.prod, null).getFirst(); - tester.deploy(previous, contentServices(14, null), Environment.prod, resourcesReductionOverride); // Allowed due to override + VespaModel previous = tester.deploy(null, contentServices(30, null), Environment.prod, null, CONTAINER_CLUSTER).getFirst(); + tester.deploy(previous, contentServices(14, null), Environment.prod, resourcesReductionOverride, CONTAINER_CLUSTER); // Allowed due to override } private void assertResourceReductionException(Exception e, NodeResources currentResources, NodeResources newResources) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/RedundancyOnFirstDeploymentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/RedundancyOnFirstDeploymentValidatorTest.java index 55a6d8cb688..30bb255746e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/RedundancyOnFirstDeploymentValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/RedundancyOnFirstDeploymentValidatorTest.java @@ -24,7 +24,7 @@ public class RedundancyOnFirstDeploymentValidatorTest { @Test void testRedundancyOnFirstDeploymentValidation() { try { - tester.deploy(null, getServices(1), Environment.prod, null); + tester.deploy(null, getServices(1), Environment.prod, null, "contentClusterId.indexing"); fail("Expected exception due to redundancy 1"); } catch (IllegalArgumentException expected) { @@ -38,7 +38,7 @@ public class RedundancyOnFirstDeploymentValidatorTest { @Test void testOverridingRedundancyOnFirstDeploymentValidation() { - tester.deploy(null, getServices(1), Environment.prod, redundancyOneOverride); // Allowed due to override + tester.deploy(null, getServices(1), Environment.prod, redundancyOneOverride, "contentClusterId.indexing"); // Allowed due to override } private static String getServices(int redundancy) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 1f2114abf98..43ea0191ca5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.collections.CollectionUtil; import com.yahoo.config.ConfigInstance; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -17,8 +19,8 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.engines.ProtonEngine; -import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; import com.yahoo.vespa.model.search.StreamingSearchCluster; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; @@ -27,10 +29,17 @@ import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.List; +import java.util.Set; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * @author baldersheim @@ -750,7 +759,8 @@ public class ContentBuilderTest extends DomBuilderTest { { String hostedXml = singleNodeContentXml(); - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true)); + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true)) + .endpoints(Set.of(new ContainerEndpoint("search.indexing", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))); VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withServices(hostedXml) .withSearchDefinition(MockApplicationPackage.MUSIC_SCHEMA) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java index 024e0922441..7af06627398 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java @@ -1,6 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -8,10 +10,14 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Set; + import static org.junit.jupiter.api.Assertions.assertNotNull; /** @@ -41,7 +47,7 @@ public class ImplicitIndexingClusterTest { VespaModel vespaModel = buildMultiTenantVespaModel(servicesXml); - ContainerCluster jdisc = vespaModel.getContainerClusters().get("jdisc"); + ContainerCluster<ApplicationContainer> jdisc = vespaModel.getContainerClusters().get("jdisc"); assertNotNull(jdisc.getDocproc(), "Docproc not added to jdisc"); assertNotNull(jdisc.getDocprocChains().allChains().getComponent("indexing"), "Indexing chain not added to jdisc"); } @@ -50,6 +56,7 @@ public class ImplicitIndexingClusterTest { ModelContext.Properties properties = new TestProperties().setMultitenant(true).setHostedVespa(true); DeployState.Builder deployStateBuilder = new DeployState.Builder() .properties(properties) + .endpoints(Set.of(new ContainerEndpoint("jdisc", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .modelHostProvisioner(new InMemoryProvisioner(6, false)); return new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index e0dd3b2180c..e7ef1a65312 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -338,6 +338,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .modelHostProvisioner(provisioner) .provisioned(provisioner.startProvisionedRecording()) .applicationPackage(applicationPackage) + .endpoints(Set.of(new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .properties(new TestProperties().setMultitenant(true) .setHostedVespa(true) .setZone(new Zone(environment, RegionName.from(region)))) @@ -548,7 +549,8 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { final var deployState = new DeployState.Builder() .applicationPackage(applicationPackage) .zone(new Zone(Environment.prod, RegionName.from("us-east-1"))) - .endpoints(Set.of(new ContainerEndpoint("comics-search", ApplicationClusterEndpoint.Scope.global, List.of("nalle", "balle")))) + .endpoints(Set.of(new ContainerEndpoint("comics-search", ApplicationClusterEndpoint.Scope.global, List.of("nalle", "balle")), + new ContainerEndpoint("comics-search", ApplicationClusterEndpoint.Scope.zone, List.of("nalle", "balle")))) .properties(new TestProperties().setHostedVespa(true)) .build(); @@ -573,6 +575,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .modelHostProvisioner(new InMemoryProvisioner(true, false, "host1.yahoo.com", "host2.yahoo.com")) .applicationPackage(applicationPackage) + .endpoints(Set.of(new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .properties(new TestProperties() .setMultitenant(true) .setHostedVespa(true)) @@ -590,6 +593,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .modelHostProvisioner(provisioner) .provisioned(provisioner.startProvisionedRecording()) .applicationPackage(applicationPackage) + .endpoints(Set.of(new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .properties(new TestProperties().setMultitenant(true) .setHostedVespa(true) .setCloudAccount(cloudAccount)) @@ -632,6 +636,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) .properties(new TestProperties().setHostedVespa(true)) + .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("c.example.com")))) .build()); AbstractConfigProducerRoot modelRoot = model.getRoot(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java index 92e28d87d8f..138bef3ae73 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java @@ -5,6 +5,8 @@ import com.yahoo.component.ComponentId; import com.yahoo.config.InnerNode; import com.yahoo.config.ModelNode; import com.yahoo.config.ModelReference; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -19,10 +21,10 @@ import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.BertEmbedder; +import com.yahoo.vespa.model.container.component.ColBertEmbedder; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.HuggingFaceEmbedder; import com.yahoo.vespa.model.container.component.HuggingFaceTokenizer; -import com.yahoo.vespa.model.container.component.ColBertEmbedder; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; @@ -35,6 +37,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -190,7 +193,11 @@ public class EmbedderTestCase { private VespaModel loadModel(Path path, boolean hosted) throws Exception { FilesApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(path.toFile()); TestProperties properties = new TestProperties().setHostedVespa(hosted); - DeployState state = new DeployState.Builder().properties(properties).applicationPackage(applicationPackage).build(); + DeployState state = new DeployState.Builder() + .properties(properties) + .endpoints(hosted ? Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com"))) : Set.of()) + .applicationPackage(applicationPackage) + .build(); return new VespaModel(state); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index 0c8547c0e5e..e60052cb2a5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -5,6 +5,8 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.collections.Pair; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -15,10 +17,12 @@ import com.yahoo.vespa.model.container.ContainerCluster; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; + import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.logging.Level; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -82,9 +86,11 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); // Need to create VespaModel to make deploy properties have effect final TestLogger logger = new TestLogger(); + Set<ContainerEndpoint> endpoints = isHosted ? Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com"))) : Set.of(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) .deployLogger(logger) + .endpoints(endpoints) .properties(new TestProperties().setHostedVespa(isHosted)) .build()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); @@ -109,9 +115,11 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); // Need to create VespaModel to make deploy properties have effect final TestLogger logger = new TestLogger(); + Set<ContainerEndpoint> endpoints = isHosted ? Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com"))) : Set.of(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) .deployLogger(logger) + .endpoints(endpoints) .properties(new TestProperties().setJvmGCOptions(featureFlagDefault).setHostedVespa(isHosted)) .build()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); @@ -249,6 +257,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(app) .deployLogger(logger) + .endpoints(properties.hostedVespa() ? Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com"))) : Set.of()) .properties(properties) .build()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index accafa032f9..d4087c0acf9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1,6 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -48,8 +50,15 @@ import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; +import java.util.Set; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class ContentClusterTest extends ContentBaseTest { @@ -382,8 +391,8 @@ public class ContentClusterTest extends ContentBaseTest { return createEnd2EndOneNode(properties, services); } - VespaModel createEnd2EndOneNode(ModelContext.Properties properties, String services) { - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(properties); + VespaModel createEnd2EndOneNode(ModelContext.Properties properties, String services, ContainerEndpoint ...containerEndpoint) { + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(properties).endpoints(Set.of(containerEndpoint)); List<String> sds = ApplicationPackageUtils.generateSchemas("type1"); return (new VespaModelCreatorWithMockPkg(null, services, sds)).create(deployStateBuilder); } @@ -1333,7 +1342,7 @@ public class ContentClusterTest extends ContentBaseTest { "<?xml version='1.0' encoding='UTF-8' ?>" + "<services version='1.0'>" + " <container id='default' version='1.0' />" + - " </services>"); + " </services>", new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com"))); assertEquals(Map.of(), noContentModel.getContentClusters()); assertNull(noContentModel.getAdmin().getClusterControllers(), "No cluster controller without content"); @@ -1348,7 +1357,7 @@ public class ContentClusterTest extends ContentBaseTest { " <document mode='index' type='type1' />" + " </documents>" + " </content>" + - " </services>"); + " </services>", new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com"))); assertNotNull(oneContentModel.getAdmin().getClusterControllers(), "Shared cluster controller with content"); String twoContentServices = "<?xml version='1.0' encoding='UTF-8' ?>" + @@ -1379,7 +1388,7 @@ public class ContentClusterTest extends ContentBaseTest { " </services>"; VespaModel twoContentModel = createEnd2EndOneNode(new TestProperties().setHostedVespa(true) .setMultitenant(true), - twoContentServices); + twoContentServices, new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com"))); assertNotNull(twoContentModel.getAdmin().getClusterControllers(), "Shared cluster controller with content"); ClusterControllerContainerCluster clusterControllers = twoContentModel.getAdmin().getClusterControllers(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java index bd49a9a7fc9..4fd61f59ed7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java @@ -1,18 +1,20 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; -import com.yahoo.config.FileReference; -import com.yahoo.config.model.ApplicationPackageTester; import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; +import com.yahoo.config.FileReference; +import com.yahoo.config.model.ApplicationPackageTester; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.vespa.model.VespaModel; @@ -22,8 +24,11 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.List; import java.util.Optional; +import java.util.Set; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Helper for testing of imported models. @@ -51,6 +56,7 @@ public class ImportedModelTester { this.modelName = modelName; this.applicationDir = applicationDir; deployState = deployStateBuilder.applicationPackage(ApplicationPackageTester.create(applicationDir.toString()).app()) + .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .modelImporters(importers) .build(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java index ed8b632e509..4c5f20a4e2f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java @@ -7,10 +7,13 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.MapConfigModelRegistry; import com.yahoo.config.model.admin.AdminModel; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.provision.Zone; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.HostResource; @@ -26,8 +29,11 @@ import org.w3c.dom.Element; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Set; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * Demonstrates how a model can be added at build time to amend another model. @@ -73,7 +79,9 @@ public class ModelAmendingTestCase { "</services>"; VespaModelTester tester = new VespaModelTester(amendingModelRepo); tester.addHosts(12); - VespaModel model = tester.createModel(services); + DeployState.Builder builder = new DeployState.Builder().endpoints(Set.of(new ContainerEndpoint("test1", ApplicationClusterEndpoint.Scope.zone, List.of("t1.example.com")), + new ContainerEndpoint("test2", ApplicationClusterEndpoint.Scope.zone, List.of("t2.example.com")))); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, builder); // Check that all hosts are amended for (HostResource host : model.getAdmin().hostSystem().getHosts()) { @@ -120,7 +128,9 @@ public class ModelAmendingTestCase { "</services>"; VespaModelTester tester = new VespaModelTester(amendingModelRepo); tester.addHosts(12); - VespaModel model = tester.createModel(services); + DeployState.Builder builder = new DeployState.Builder().endpoints(Set.of(new ContainerEndpoint("test1", ApplicationClusterEndpoint.Scope.zone, List.of("t1.example.com")), + new ContainerEndpoint("test2", ApplicationClusterEndpoint.Scope.zone, List.of("t2.example.com")))); + VespaModel model = tester.createModel(Zone.defaultZone(), services, true, builder); // Check that all hosts are amended for (HostResource host : model.getAdmin().hostSystem().getHosts()) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index c6a1562f906..446a48afd20 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.model.test; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -21,12 +23,14 @@ import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; + import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import static com.yahoo.config.provision.NodeResources.Architecture; import static com.yahoo.vespa.model.test.utils.ApplicationPackageUtils.generateSchemas; @@ -191,7 +195,7 @@ public class VespaModelTester { * * @param services the services xml string * @param hosts the hosts xml string, or null if none - * @param useMaxResources false to use the minmal resources (when given a range), true to use max + * @param useMaxResources false to use the minimal resources (when given a range), true to use max * @param failOnOutOfCapacity whether we should get an exception when not enough hosts of the requested flavor * is available or if we should just silently receive a smaller allocation * @return the resulting model @@ -200,10 +204,11 @@ public class VespaModelTester { boolean alwaysReturnOneNode, NodeResources defaultResources, int startIndexForClusters, Optional<VespaModel> previousModel, - DeployState.Builder deployStatebuilder, String ... retiredHostNames) { + DeployState.Builder deployStateBuilder, String ... retiredHostNames) { VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(hosts, services, generateSchemas("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; + Set<ContainerEndpoint> containerEndpoints = deployStateBuilder.build().getEndpoints(); if (hosted) { InMemoryProvisioner provisioner = new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, @@ -215,6 +220,9 @@ public class VespaModelTester { retiredHostNames); provisioner.setEnvironment(zone.environment()); this.provisioner = new ProvisionerAdapter(provisioner); + if (containerEndpoints.isEmpty()) { + containerEndpoints = Set.of(new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com"))); + } } else { provisioner = new SingleNodeProvisioner(); } @@ -226,10 +234,11 @@ public class VespaModelTester { .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver) .setAdminClusterNodeResourcesArchitecture(adminClusterArchitecture); - DeployState.Builder deployState = deployStatebuilder + DeployState.Builder deployState = deployStateBuilder .applicationPackage(appPkg) .modelHostProvisioner(provisioner) .properties(properties) + .endpoints(containerEndpoints) .zone(zone); previousModel.ifPresent(deployState::previousModel); return modelCreatorWithMockPkg.create(false, deployState.build(), configModelRegistry); diff --git a/configdefinitions/src/vespa/rank-profiles.def b/configdefinitions/src/vespa/rank-profiles.def index 7d79bbe48f3..f8a6691bf30 100644 --- a/configdefinitions/src/vespa/rank-profiles.def +++ b/configdefinitions/src/vespa/rank-profiles.def @@ -10,3 +10,14 @@ rankprofile[].fef.property[].name string ## the value of a generic property available to feature plugins rankprofile[].fef.property[].value string +## output from cross-hits normalizing function +rankprofile[].normalizer[].name string + +## input to cross-hits normalizing function +rankprofile[].normalizer[].input string + +## type of cross-hits normalizing function +rankprofile[].normalizer[].algo enum { LINEAR, RRANK } default=LINEAR + +## extra "k" param (if applicable) +rankprofile[].normalizer[].kparam double default=60.0 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 225bcb8dbed..67a1a335067 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -140,7 +140,19 @@ public class DeployTester { * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ public PrepareResult deployApp(String applicationPath, String vespaVersion) { - return deployApp(applicationPath, new PrepareParams.Builder().vespaVersion(vespaVersion)); + String endpoints = """ + [ + { + "clusterId": "container", + "names": [ + "c.example.com" + ], + "scope": "zone", + "routingMethod": "exclusive" + } + ] + """; + return deployApp(applicationPath, new PrepareParams.Builder().containerEndpoints(endpoints).vespaVersion(vespaVersion)); } /** diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java index ec9bd3f7245..44344b6d394 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java @@ -63,10 +63,22 @@ public class HostedDeployNodeAllocationTest { .provisioner(new MockProvisioner().hostProvisioner(provisioner)) .hostedConfigserverConfig(Zone.defaultZone()) .build(); - + String endpoints = """ + [ + { + "clusterId": "container", + "names": [ + "c.example.com" + ], + "scope": "zone", + "routingMethod": "exclusive" + } + ] + """; try { tester.deployApp("src/test/apps/hosted/", new PrepareParams.Builder() .vespaVersion("7.3") + .containerEndpoints(endpoints) .quota(new Quota(Optional.of(4), Optional.of(0)))); fail("Expected to get a QuotaExceededException"); } catch (QuotaExceededException e) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java index 0021e5f4b6a..5fd23a95eed 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java @@ -63,7 +63,19 @@ public class TenantsMaintainerTest { } private PrepareParams.Builder prepareParams(TenantName tenantName) { - return new PrepareParams.Builder().applicationId(applicationId(tenantName)); + String endpoints = """ + [ + { + "clusterId": "container", + "names": [ + "c.example.com" + ], + "scope": "zone", + "routingMethod": "exclusive" + } + ] + """; + return new PrepareParams.Builder().containerEndpoints(endpoints).applicationId(applicationId(tenantName)); } private ApplicationId applicationId(TenantName tenantName) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 9954a40512f..e1ad6bf51f0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -81,7 +81,7 @@ public class LbServicesProducerTest { private LbServicesConfig createModelAndGetLbServicesConfig(RegionName regionName) { Zone zone = new Zone(Environment.prod, regionName); - Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder().zone(zone)); + Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder().endpoints(Set.of(new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("md.example.com")))).zone(zone)); return getLbServicesConfig(new Zone(Environment.prod, regionName), testModel); } @@ -126,7 +126,7 @@ public class LbServicesProducerTest { @Test public void testRoutingConfigForTesterApplication() { - Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder()); + Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder().endpoints(Set.of(new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("md.example.com"))))); // No config for tester application assertNull(getLbServicesConfig(Zone.defaultZone(), testModel) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java index 29515a6c872..b64c0417043 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.config.server.provision; import com.yahoo.cloud.config.ModelConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; @@ -16,6 +18,8 @@ import org.xml.sax.SAXException; import java.io.File; import java.io.IOException; +import java.util.List; +import java.util.Set; import static org.junit.Assert.assertEquals; @@ -50,6 +54,7 @@ public class StaticProvisionerTest { DeployState deployState = new DeployState.Builder() .applicationPackage(app) .modelHostProvisioner(provisioner) + .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("c.example.com")))) .properties(new TestProperties() .setMultitenant(true) .setHostedVespa(true)) diff --git a/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java b/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java index b203d7a9fb9..fef23f3cc27 100644 --- a/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java +++ b/container-core/src/main/java/com/yahoo/component/chain/dependencies/ordering/ChainBuilder.java @@ -46,7 +46,7 @@ public class ChainBuilder<T extends ChainedComponent> { public ChainBuilder(ComponentId id) { this.id = id; - allPhase = addPhase(new Phase("*", set("*"), Collections.<String>emptySet())); + allPhase = addPhase(new Phase("*", set("*"), Set.of())); } private Set<String> set(String... s) { diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadpoolImpl.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadpoolImpl.java index befbda28ac0..f92d218390f 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadpoolImpl.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadpoolImpl.java @@ -54,7 +54,7 @@ public class ContainerThreadpoolImpl extends AbstractComponent implements AutoCl createQueue(queueSize), ThreadFactoryFactory.getThreadFactory(name), threadPoolMetric); - // Prestart needed, if not all threads will be created by the fist N tasks and hence they might also + // Pre-start needed, if not all threads will be created by the fist N tasks and hence they might also // get the dreaded thread locals initialized even if they will never run. // That counters what we want to achieve with the Q that will prefer thread locality. executor.prestartAllCoreThreads(); diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java index e3c2c78abec..cee2cc54b5b 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/WorkerCompletionTimingThreadPoolExecutor.java @@ -8,8 +8,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; /** - * A thread pool executor which maintains the last time a worker completed - * package private for testing + * A thread pool executor which maintains the last time a worker completed. * * @author Steinar Knutsen * @author baldersheim diff --git a/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java b/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java index 315dc21ec38..3c44ba2eaa6 100644 --- a/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java +++ b/container-core/src/main/java/com/yahoo/container/protect/ProcessTerminator.java @@ -4,8 +4,8 @@ package com.yahoo.container.protect; import com.yahoo.protect.Process; /** - * An injectable terminator of the Java vm. - * Components that encounters conditions where the vm should be terminated should + * An injectable terminator of the Java VM. + * Components that encounter conditions where the VM should be terminated should * request an instance of this injected. That makes termination testable * as tests can create subclasses of this which register the termination request * rather than terminating. diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index dca83e8e556..623d11cc473 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -29,6 +29,7 @@ import com.yahoo.jdisc.application.DeactivatedContainer; import com.yahoo.jdisc.application.GuiceRepository; import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.http.server.jetty.JettyHttpServer; import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; import com.yahoo.jrt.Acceptor; @@ -41,6 +42,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.Register; import com.yahoo.jrt.slobrok.api.SlobrokList; +import com.yahoo.messagebus.jdisc.MbusServer; import com.yahoo.messagebus.network.rpc.SlobrokConfigSubscriber; import com.yahoo.net.HostName; import com.yahoo.security.tls.Capability; @@ -53,8 +55,10 @@ import java.security.Provider; import java.security.Security; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; @@ -377,20 +381,20 @@ public final class ConfiguredApplication implements Application { synchronized (monitor) { Set<ServerProvider> serversToClose = createIdentityHashSet(startedServers); serversToClose.removeAll(currentServers); - for (ServerProvider server : currentServers) { + for (ServerProvider server : ordered(currentServers, MbusServer.class, JettyHttpServer.class)) { if ( ! startedServers.contains(server) && server.isMultiplexed()) { server.start(); startedServers.add(server); } } - if (serversToClose.size() > 0) { + if ( ! serversToClose.isEmpty()) { log.info(String.format("Closing %d server instances", serversToClose.size())); - for (ServerProvider server : serversToClose) { + for (ServerProvider server : ordered(serversToClose, JettyHttpServer.class, MbusServer.class)) { server.close(); startedServers.remove(server); } } - for (ServerProvider server : currentServers) { + for (ServerProvider server : ordered(currentServers, MbusServer.class, JettyHttpServer.class)) { if ( ! startedServers.contains(server)) { server.start(); startedServers.add(server); @@ -525,6 +529,22 @@ public final class ConfiguredApplication implements Application { } } + /** Returns a list with the given elements, ordered by the enumerated classes, ordering more specific matches first. */ + @SafeVarargs + static <T> List<T> ordered(Collection<T> items, Class<? extends T>... order) { + List<T> ordered = new ArrayList<>(items); + ordered.sort(Comparator.comparingInt(item -> { + int best = order.length; + for (int i = 0; i < order.length; i++) { + if ( order[i].isInstance(item) + && ( best == order.length + || order[best].isAssignableFrom(order[i]))) best = i; + } + return best; + })); + return ordered; + } + private static <E> Set<E> createIdentityHashSet() { return Collections.newSetFromMap(new IdentityHashMap<>()); } diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/ConfiguredApplicationTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/ConfiguredApplicationTest.java new file mode 100644 index 00000000000..555d43f7697 --- /dev/null +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/ConfiguredApplicationTest.java @@ -0,0 +1,27 @@ +package com.yahoo.container.jdisc; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static com.yahoo.container.jdisc.ConfiguredApplication.ordered; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ConfiguredApplicationTest { + + @Test + void testSorting() { + class A { @Override public String toString() { return getClass().getSimpleName(); } } + class B extends A { } + class C extends B { } + class D extends B { } + + A a = new A(), b = new B(), c = new C(), d = new D(), e = new D() { @Override public String toString() { return "E"; } }; + List<A> s = List.of(a, b, c, d, e); + assertEquals(List.of(a, b, c, d, e), ordered(s, A.class, B.class, C.class, D.class)); + assertEquals(List.of(d, e, c, b, a), ordered(s, D.class, C.class, B.class, A.class)); + assertEquals(List.of(e, c, a, b, d), ordered(s, e.getClass(), C.class, A.class)); + assertEquals(List.of(d, e, b, c, a), ordered(s, D.class, B.class)); + } + +} diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/ShutdownDeadlineTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/ShutdownDeadlineTest.java index 6351f73502e..6a846c6f3ae 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/ShutdownDeadlineTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/ShutdownDeadlineTest.java @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; import org.junit.jupiter.api.Test; diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java index 3949d713768..4c21489ded2 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java @@ -48,8 +48,8 @@ public final class MbusServer extends AbstractResource implements ServerProvider @Override public void start() { log.log(Level.FINE, "Starting message bus server."); - session.connect(); runState.set(State.RUNNING); + session.connect(); } @Override @@ -66,7 +66,7 @@ public final class MbusServer extends AbstractResource implements ServerProvider @Override protected void destroy() { - log.log(Level.FINE, "Destroying message bus server."); + log.log(Level.INFO, "Destroying message bus server: " + session.name()); runState.set(State.STOPPED); sessionReference.close(); } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java index 87a9cb61a0e..cc428aec7a7 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java @@ -22,12 +22,13 @@ import java.util.Iterator; @Before(GroupingExecutor.COMPONENT_NAME) public class FlatteningSearcher extends Searcher { - private final CompoundName flatten = CompoundName.from("grouping.flatten"); + private final CompoundName groupingFlatten = CompoundName.from("grouping.flatten"); + private final CompoundName flatten = CompoundName.from("flatten"); @Override public Result search(Query query, Execution execution) { + if ( ! query.properties().getBoolean(groupingFlatten, true)) return execution.search(query); if ( ! query.properties().getBoolean(flatten, true)) return execution.search(query); - if ( ! query.properties().getBoolean("flatten", true)) return execution.search(query); query.trace("Flattening groups", 2); int originalHits = query.getHits(); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/DummyEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/DummyEvaluator.java new file mode 100644 index 00000000000..e83a308d99c --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/DummyEvaluator.java @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; +import com.yahoo.tensor.Tensor; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +class DummyEvaluator implements Evaluator { + + private final String input; + private Tensor result = null; + + DummyEvaluator(String input) { + this.input = input; + } + + @Override + public Evaluator bind(String name, Tensor value) { + result = value; + return this; + } + + @Override + public double evaluateScore() { + return result.asDouble(); + } + + @Override + public String toString() { + return "DummyEvaluator(" + input + ")"; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java index 9dab252e9a4..83f9d0e2704 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java @@ -3,11 +3,9 @@ package com.yahoo.search.ranking; import com.yahoo.tensor.Tensor; -import java.util.Collection; +import java.util.List; interface Evaluator { - Collection<String> needInputs(); - Evaluator bind(String name, Tensor value); double evaluateScore(); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/FunEvalSpec.java b/container-search/src/main/java/com/yahoo/search/ranking/FunEvalSpec.java new file mode 100644 index 00000000000..df9c509dd82 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/FunEvalSpec.java @@ -0,0 +1,7 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import java.util.List; +import java.util.function.Supplier; + +record FunEvalSpec(Supplier<Evaluator> evalSource, List<String> fromQuery, List<String> fromMF) {} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java index 01ea5e3ebd5..6e30a81eebc 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java @@ -5,7 +5,6 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.query.Sorting; -import com.yahoo.search.ranking.RankProfilesEvaluator.GlobalPhaseData; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.FeatureData; import com.yahoo.search.result.Hit; @@ -33,8 +32,8 @@ public class GlobalPhaseRanker { } public Optional<ErrorMessage> validateNoSorting(Query query, String schema) { - var data = globalPhaseDataFor(query, schema).orElse(null); - if (data == null) return Optional.empty(); + var setup = globalPhaseSetupFor(query, schema).orElse(null); + if (setup == null) return Optional.empty(); var sorting = query.getRanking().getSorting(); if (sorting == null || sorting.fieldOrders() == null) return Optional.empty(); for (var fieldOrder : sorting.fieldOrders()) { @@ -47,23 +46,33 @@ public class GlobalPhaseRanker { } public void rerankHits(Query query, Result result, String schema) { - var data = globalPhaseDataFor(query, schema).orElse(null); - if (data == null) return; - var functionEvaluatorSource = data.functionEvaluatorSource(); - var prepared = findFromQuery(query, data.needInputs()); + var setup = globalPhaseSetupFor(query, schema).orElse(null); + if (setup == null) return; + var mainSpec = setup.globalPhaseEvalSpec; + var mainSrc = withQueryPrep(mainSpec.evalSource(), mainSpec.fromQuery(), query); + int rerankCount = setup.rerankCount; + var normalizers = new ArrayList<NormalizerContext>(); + for (var nSetup : setup.normalizers) { + var normSpec = nSetup.inputEvalSpec(); + var normEvalSrc = withQueryPrep(normSpec.evalSource(), normSpec.fromQuery(), query); + normalizers.add(new NormalizerContext(nSetup.name(), nSetup.supplier().get(), normEvalSrc, normSpec.fromMF())); + } + var rescorer = new HitRescorer(mainSrc, mainSpec.fromMF(), normalizers); + var reranker = new ResultReranker(rescorer, rerankCount); + reranker.rerankHits(result); + hideImplicitMatchFeatures(result, setup.matchFeaturesToHide); + } + + static Supplier<Evaluator> withQueryPrep(Supplier<Evaluator> evalSource, List<String> queryFeatures, Query query) { + var prepared = PreparedInput.findFromQuery(query, queryFeatures); Supplier<Evaluator> supplier = () -> { - var evaluator = functionEvaluatorSource.get(); - var simple = new SimpleEvaluator(evaluator); + var evaluator = evalSource.get(); for (var entry : prepared) { - simple.bind(entry.name(), entry.value()); + evaluator.bind(entry.name(), entry.value()); } - return simple; + return evaluator; }; - int rerankCount = data.rerankCount(); - if (rerankCount < 0) - rerankCount = 100; - ResultReranker.rerankHits(result, new HitRescorer(supplier), rerankCount); - hideImplicitMatchFeatures(result, data.matchFeaturesToHide()); + return supplier; } private void hideImplicitMatchFeatures(Result result, Collection<String> namesToHide) { @@ -87,44 +96,9 @@ public class GlobalPhaseRanker { } } - private Optional<GlobalPhaseData> globalPhaseDataFor(Query query, String schema) { + private Optional<GlobalPhaseSetup> globalPhaseSetupFor(Query query, String schema) { return factory.evaluatorForSchema(schema) - .flatMap(evaluator -> evaluator.getGlobalPhaseData(query.getRanking().getProfile())); - } - - record NameAndValue(String name, Tensor value) { } - - /* do this only once per query: */ - List<NameAndValue> findFromQuery(Query query, List<String> needInputs) { - List<NameAndValue> result = new ArrayList<>(); - var ranking = query.getRanking(); - var rankFeatures = ranking.getFeatures(); - var rankProps = ranking.getProperties().asMap(); - for (String needed : needInputs) { - var optRef = com.yahoo.searchlib.rankingexpression.Reference.simple(needed); - if (optRef.isEmpty()) continue; - var ref = optRef.get(); - if (ref.name().equals("constant")) { - // XXX in theory, we should be able to avoid this - result.add(new NameAndValue(needed, null)); - continue; - } - if (ref.isSimple() && ref.name().equals("query")) { - String queryFeatureName = ref.simpleArgument().get(); - // searchers are recommended to place query features here: - var feature = rankFeatures.getTensor(queryFeatureName); - if (feature.isPresent()) { - result.add(new NameAndValue(needed, feature.get())); - } else { - // but other ways of setting query features end up in the properties: - var objList = rankProps.get(queryFeatureName); - if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) { - result.add(new NameAndValue(needed, t)); - } - } - } - } - return result; + .flatMap(evaluator -> evaluator.getGlobalPhaseSetup(query.getRanking().getProfile())); } } diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseSetup.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseSetup.java new file mode 100644 index 00000000000..31a676e4c8e --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseSetup.java @@ -0,0 +1,153 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; + +import com.yahoo.vespa.config.search.RankProfilesConfig; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.Map; +import java.util.HashMap; +import java.util.function.Supplier; + +class GlobalPhaseSetup { + + final FunEvalSpec globalPhaseEvalSpec; + final int rerankCount; + final Collection<String> matchFeaturesToHide; + final List<NormalizerSetup> normalizers; + + GlobalPhaseSetup(FunEvalSpec globalPhaseEvalSpec, + final int rerankCount, + Collection<String> matchFeaturesToHide, + List<NormalizerSetup> normalizers) + { + this.globalPhaseEvalSpec = globalPhaseEvalSpec; + this.rerankCount = rerankCount; + this.matchFeaturesToHide = matchFeaturesToHide; + this.normalizers = normalizers; + } + + static GlobalPhaseSetup maybeMakeSetup(RankProfilesConfig.Rankprofile rp, RankProfilesEvaluator modelEvaluator) { + var model = modelEvaluator.modelForRankProfile(rp.name()); + Map<String, RankProfilesConfig.Rankprofile.Normalizer> availableNormalizers = new HashMap<>(); + for (var n : rp.normalizer()) { + availableNormalizers.put(n.name(), n); + } + Supplier<FunctionEvaluator> functionEvaluatorSource = null; + int rerankCount = -1; + Set<String> namesToHide = new HashSet<>(); + Set<String> matchFeatures = new HashSet<>(); + Map<String, String> renameFeatures = new HashMap<>(); + String toRename = null; + for (var prop : rp.fef().property()) { + if (prop.name().equals("vespa.globalphase.rerankcount")) { + rerankCount = Integer.valueOf(prop.value()); + } + if (prop.name().equals("vespa.rank.globalphase")) { + functionEvaluatorSource = () -> model.evaluatorOf("globalphase"); + } + if (prop.name().equals("vespa.hidden.matchfeature")) { + namesToHide.add(prop.value()); + } + if (prop.name().equals("vespa.match.feature")) { + matchFeatures.add(prop.value()); + } + if (prop.name().equals("vespa.feature.rename")) { + if (toRename == null) { + toRename = prop.value(); + } else { + renameFeatures.put(toRename, prop.value()); + toRename = null; + } + } + } + for (var entry : renameFeatures.entrySet()) { + String old = entry.getKey(); + if (matchFeatures.contains(old)) { + matchFeatures.remove(old); + matchFeatures.add(entry.getValue()); + } + } + if (rerankCount < 0) { + rerankCount = 100; + } + if (functionEvaluatorSource != null) { + var evaluator = functionEvaluatorSource.get(); + var allInputs = List.copyOf(evaluator.function().arguments()); + List<String> fromMF = new ArrayList<>(); + List<String> fromQuery = new ArrayList<>(); + List<NormalizerSetup> normalizers = new ArrayList<>(); + for (var input : allInputs) { + String queryFeatureName = asQueryFeature(input); + if (queryFeatureName != null) { + fromQuery.add(queryFeatureName); + } else if (availableNormalizers.containsKey(input)) { + var cfg = availableNormalizers.get(input); + String normInput = cfg.input(); + if (matchFeatures.contains(normInput)) { + Supplier<Evaluator> normSource = () -> new DummyEvaluator(normInput); + normalizers.add(makeNormalizerSetup(cfg, matchFeatures, normSource, List.of(normInput), rerankCount)); + } else { + Supplier<FunctionEvaluator> normSource = () -> model.evaluatorOf(normInput); + var normInputs = List.copyOf(normSource.get().function().arguments()); + var normSupplier = SimpleEvaluator.wrap(normSource); + normalizers.add(makeNormalizerSetup(cfg, matchFeatures, normSupplier, normInputs, rerankCount)); + } + } else if (matchFeatures.contains(input)) { + fromMF.add(input); + } else { + throw new IllegalArgumentException("Bad config, missing global-phase input: " + input); + } + } + Supplier<Evaluator> supplier = SimpleEvaluator.wrap(functionEvaluatorSource); + var gfun = new FunEvalSpec(supplier, fromQuery, fromMF); + return new GlobalPhaseSetup(gfun, rerankCount, namesToHide, normalizers); + } + return null; + } + + private static NormalizerSetup makeNormalizerSetup(RankProfilesConfig.Rankprofile.Normalizer cfg, + Set<String> matchFeatures, + Supplier<Evaluator> evalSupplier, + List<String> normInputs, + int rerankCount) + { + List<String> fromQuery = new ArrayList<>(); + List<String> fromMF = new ArrayList<>(); + for (var input : normInputs) { + String queryFeatureName = asQueryFeature(input); + if (queryFeatureName != null) { + fromQuery.add(queryFeatureName); + } else if (matchFeatures.contains(input)) { + fromMF.add(input); + } else { + throw new IllegalArgumentException("Bad config, missing normalizer input: " + input); + } + } + var fun = new FunEvalSpec(evalSupplier, fromQuery, fromMF); + return new NormalizerSetup(cfg.name(), makeNormalizerSupplier(cfg, rerankCount), fun); + } + + private static Supplier<Normalizer> makeNormalizerSupplier(RankProfilesConfig.Rankprofile.Normalizer cfg, int rerankCount) { + return switch (cfg.algo()) { + case LINEAR -> () -> new LinearNormalizer(rerankCount); + case RRANK -> () -> new ReciprocalRankNormalizer(rerankCount, cfg.kparam()); + }; + } + + static String asQueryFeature(String input) { + var optRef = com.yahoo.searchlib.rankingexpression.Reference.simple(input); + if (optRef.isPresent()) { + var ref = optRef.get(); + if (ref.isSimple() && ref.name().equals("query")) { + return ref.simpleArgument().get(); + } + } + return null; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java index f6519158e88..fee4f5b4160 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java @@ -3,55 +3,61 @@ package com.yahoo.search.ranking; import com.yahoo.search.result.FeatureData; import com.yahoo.search.result.Hit; -import static com.yahoo.searchlib.rankingexpression.Reference.RANKING_EXPRESSION_WRAPPER; +import com.yahoo.tensor.Tensor; +import java.util.List; import java.util.function.Supplier; import java.util.logging.Logger; class HitRescorer { private static final Logger logger = Logger.getLogger(HitRescorer.class.getName()); - - private final Supplier<Evaluator> evaluatorSource; - public HitRescorer(Supplier<Evaluator> evaluatorSource) { - this.evaluatorSource = evaluatorSource; + private final Supplier<Evaluator> mainEvalSrc; + private final List<String> mainFromMF; + private final List<NormalizerContext> normalizers; + + public HitRescorer(Supplier<Evaluator> mainEvalSrc, List<String> mainFromMF, List<NormalizerContext> normalizers) { + this.mainEvalSrc = mainEvalSrc; + this.mainFromMF = mainFromMF; + this.normalizers = normalizers; } - boolean rescoreHit(Hit hit) { - var features = hit.getField("matchfeatures"); - if (features instanceof FeatureData matchFeatures) { - var scorer = evaluatorSource.get(); - for (String argName : scorer.needInputs()) { - var asTensor = matchFeatures.getTensor(argName); - if (asTensor == null) { - asTensor = matchFeatures.getTensor(alternate(argName)); - } - if (asTensor != null) { - scorer.bind(argName, asTensor); - } else { - logger.warning("Missing match-feature for Evaluator argument: " + argName); - return false; - } - } - double newScore = scorer.evaluateScore(); - hit.setRelevance(newScore); - return true; - } else { - logger.warning("Hit without match-features: " + hit); - return false; + void preprocess(WrappedHit wrapped) { + for (var n : normalizers) { + var scorer = n.evalSource().get(); + double val = evalScorer(wrapped, scorer, n.fromMF()); + wrapped.setIdx(n.normalizer().addInput(val)); + } + } + + void runNormalizers() { + for (var n : normalizers) { + n.normalizer().normalize(); } } - private static final String RE_PREFIX = RANKING_EXPRESSION_WRAPPER + "("; - private static final String RE_SUFFIX = ")"; - private static final int RE_PRE_LEN = RE_PREFIX.length(); - private static final int RE_SUF_LEN = RE_SUFFIX.length(); + double rescoreHit(WrappedHit wrapped) { + var scorer = mainEvalSrc.get(); + for (var n : normalizers) { + double normalizedValue = n.normalizer().getOutput(wrapped.getIdx()); + scorer.bind(n.name(), Tensor.from(normalizedValue)); + } + double newScore = evalScorer(wrapped, scorer, mainFromMF); + wrapped.setScore(newScore); + return newScore; + } - static String alternate(String argName) { - if (argName.startsWith(RE_PREFIX) && argName.endsWith(RE_SUFFIX)) { - return argName.substring(RE_PRE_LEN, argName.length() - RE_SUF_LEN); + private static double evalScorer(WrappedHit wrapped, Evaluator scorer, List<String> fromMF) { + for (String argName : fromMF) { + var asTensor = wrapped.getTensor(argName); + if (asTensor != null) { + scorer.bind(argName, asTensor); + } else { + logger.warning("Missing match-feature for Evaluator argument: " + argName); + return 0.0; + } } - return argName; + return scorer.evaluateScore(); } } diff --git a/container-search/src/main/java/com/yahoo/search/ranking/LinearNormalizer.java b/container-search/src/main/java/com/yahoo/search/ranking/LinearNormalizer.java index a3fb86bb9b5..2cdba9d6361 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/LinearNormalizer.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/LinearNormalizer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; class LinearNormalizer extends Normalizer { diff --git a/container-search/src/main/java/com/yahoo/search/ranking/Normalizer.java b/container-search/src/main/java/com/yahoo/search/ranking/Normalizer.java index 269d4e6ed11..eb81d0555b3 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/Normalizer.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/Normalizer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; abstract class Normalizer { diff --git a/container-search/src/main/java/com/yahoo/search/ranking/NormalizerContext.java b/container-search/src/main/java/com/yahoo/search/ranking/NormalizerContext.java new file mode 100644 index 00000000000..9438b5ea824 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/NormalizerContext.java @@ -0,0 +1,7 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import java.util.List; +import java.util.function.Supplier; + +record NormalizerContext(String name, Normalizer normalizer, Supplier<Evaluator> evalSource, List<String> fromMF) {} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/NormalizerSetup.java b/container-search/src/main/java/com/yahoo/search/ranking/NormalizerSetup.java new file mode 100644 index 00000000000..32fbb3190fc --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/NormalizerSetup.java @@ -0,0 +1,6 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import java.util.function.Supplier; + +record NormalizerSetup(String name, Supplier<Normalizer> supplier, FunEvalSpec inputEvalSpec) {} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java b/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java new file mode 100644 index 00000000000..5ab2d7160f9 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.query.Sorting; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.tensor.Tensor; +import com.yahoo.data.access.helpers.MatchFeatureData; +import com.yahoo.data.access.helpers.MatchFeatureFilter; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.logging.Logger; + +record PreparedInput(String name, Tensor value) { + + static List<PreparedInput> findFromQuery(Query query, Collection<String> queryFeatures) { + List<PreparedInput> result = new ArrayList<>(); + var ranking = query.getRanking(); + var rankFeatures = ranking.getFeatures(); + var rankProps = ranking.getProperties().asMap(); + for (String queryFeatureName : queryFeatures) { + String needed = "query(" + queryFeatureName + ")"; + // searchers are recommended to place query features here: + var feature = rankFeatures.getTensor(queryFeatureName); + if (feature.isPresent()) { + result.add(new PreparedInput(needed, feature.get())); + } else { + // but other ways of setting query features end up in the properties: + var objList = rankProps.get(queryFeatureName); + if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) { + result.add(new PreparedInput(needed, t)); + } else { + throw new IllegalArgumentException("missing query feature: " + queryFeatureName); + } + } + } + return result; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RangeAdjuster.java b/container-search/src/main/java/com/yahoo/search/ranking/RangeAdjuster.java new file mode 100644 index 00000000000..6881eece620 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RangeAdjuster.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +// scale and adjust the score according to the range +// of the original and final score values to avoid that +// a score from the backend is larger than finalScores_low +class RangeAdjuster { + private double initialScores_high = -Double.MAX_VALUE; + private double initialScores_low = Double.MAX_VALUE; + private double finalScores_high = -Double.MAX_VALUE; + private double finalScores_low = Double.MAX_VALUE; + + boolean rescaleNeeded() { + return (initialScores_low > finalScores_low + && + initialScores_high >= initialScores_low + && + finalScores_high >= finalScores_low); + } + void withInitialScore(double score) { + if (score < initialScores_low) initialScores_low = score; + if (score > initialScores_high) initialScores_high = score; + } + void withFinalScore(double score) { + if (score < finalScores_low) finalScores_low = score; + if (score > finalScores_high) finalScores_high = score; + } + private double initialRange() { + double r = initialScores_high - initialScores_low; + if (r < 1.0) r = 1.0; + return r; + } + private double finalRange() { + double r = finalScores_high - finalScores_low; + if (r < 1.0) r = 1.0; + return r; + } + double scale() { return finalRange() / initialRange(); } + double bias() { return finalScores_low - initialScores_low * scale(); } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java index 353c88d374a..0ebb98af60e 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java @@ -63,40 +63,17 @@ public class RankProfilesEvaluator extends AbstractComponent { return modelForRankProfile(rankProfile).evaluatorOf(functionName); } - static record GlobalPhaseData(Supplier<FunctionEvaluator> functionEvaluatorSource, - Collection<String> matchFeaturesToHide, - int rerankCount, - List<String> needInputs) {} + private Map<String, GlobalPhaseSetup> profilesWithGlobalPhase = new HashMap<>(); - private Map<String, GlobalPhaseData> profilesWithGlobalPhase = new HashMap<>(); - - Optional<GlobalPhaseData> getGlobalPhaseData(String rankProfile) { + Optional<GlobalPhaseSetup> getGlobalPhaseSetup(String rankProfile) { return Optional.ofNullable(profilesWithGlobalPhase.get(rankProfile)); } private void extractGlobalPhaseData(RankProfilesConfig rankProfilesConfig) { for (var rp : rankProfilesConfig.rankprofile()) { - String name = rp.name(); - Supplier<FunctionEvaluator> functionEvaluatorSource = null; - int rerankCount = -1; - List<String> needInputs = null; - Set<String> namesToHide = new HashSet<>(); - for (var prop : rp.fef().property()) { - if (prop.name().equals("vespa.globalphase.rerankcount")) { - rerankCount = Integer.valueOf(prop.value()); - } - if (prop.name().equals("vespa.rank.globalphase")) { - var model = modelForRankProfile(name); - functionEvaluatorSource = () -> model.evaluatorOf("globalphase"); - var evaluator = functionEvaluatorSource.get(); - needInputs = List.copyOf(evaluator.function().arguments()); - } - if (prop.name().equals("vespa.hidden.matchfeature")) { - namesToHide.add(prop.value()); - } - } - if (functionEvaluatorSource != null && needInputs != null) { - profilesWithGlobalPhase.put(name, new GlobalPhaseData(functionEvaluatorSource, namesToHide, rerankCount, needInputs)); + var setup = GlobalPhaseSetup.maybeMakeSetup(rp, this); + if (setup != null) { + profilesWithGlobalPhase.put(rp.name(), setup); } } } diff --git a/container-search/src/main/java/com/yahoo/search/ranking/ReciprocalRankNormalizer.java b/container-search/src/main/java/com/yahoo/search/ranking/ReciprocalRankNormalizer.java index 6716485e343..fca920a7a65 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/ReciprocalRankNormalizer.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/ReciprocalRankNormalizer.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; import java.util.Arrays; diff --git a/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java index d92068cd8d9..2e9edd6de3a 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java @@ -14,80 +14,72 @@ class ResultReranker { private static final Logger logger = Logger.getLogger(ResultReranker.class.getName()); - // scale and adjust the score according to the range - // of the original and final score values to avoid that - // a score from the backend is larger than finalScores_low - static class Ranges { - private double initialScores_high = -Double.MAX_VALUE; - private double initialScores_low = Double.MAX_VALUE; - private double finalScores_high = -Double.MAX_VALUE; - private double finalScores_low = Double.MAX_VALUE; + private final HitRescorer hitRescorer; + private final int rerankCount; + private final List<WrappedHit> hitsToRescore = new ArrayList<>(); + private final RangeAdjuster ranges = new RangeAdjuster(); - boolean rescaleNeeded() { - return (initialScores_low > finalScores_low - && - initialScores_high >= initialScores_low - && - finalScores_high >= finalScores_low); - } - void withInitialScore(double score) { - if (score < initialScores_low) initialScores_low = score; - if (score > initialScores_high) initialScores_high = score; - } - void withFinalScore(double score) { - if (score < finalScores_low) finalScores_low = score; - if (score > finalScores_high) finalScores_high = score; - } - private double initialRange() { - double r = initialScores_high - initialScores_low; - if (r < 1.0) r = 1.0; - return r; - } - private double finalRange() { - double r = finalScores_high - finalScores_low; - if (r < 1.0) r = 1.0; - return r; - } - double scale() { return finalRange() / initialRange(); } - double bias() { return finalScores_low - initialScores_low * scale(); } + ResultReranker(HitRescorer hitRescorer, int rerankCount) { + this.hitRescorer = hitRescorer; + this.rerankCount = rerankCount; } - static void rerankHits(Result result, HitRescorer hitRescorer, int rerankCount) { - List<Hit> hitsToRescore = new ArrayList<>(); - // consider doing recursive iteration explicitly instead of using deepIterator? + void rerankHits(Result result) { + gatherHits(result); + runPreProcessing(); + hitRescorer.runNormalizers(); + runProcessing(); + runPostProcessing(); + result.hits().sort(); + } + + private void gatherHits(Result result) { for (var iterator = result.hits().deepIterator(); iterator.hasNext();) { Hit hit = iterator.next(); if (hit.isMeta() || hit instanceof HitGroup) { continue; } // what about hits inside grouping results? - // they are inside GroupingListHit, we won't recurse into it; so we won't see them. - hitsToRescore.add(hit); + // they did not show up here during manual testing. + var wrapped = WrappedHit.from(hit); + if (wrapped != null) hitsToRescore.add(wrapped); } + } + + private void runPreProcessing() { // we can't be 100% certain that hits were sorted according to relevance: hitsToRescore.sort(Comparator.naturalOrder()); - var ranges = new Ranges(); - for (var iterator = hitsToRescore.iterator(); rerankCount > 0 && iterator.hasNext(); ) { - Hit hit = iterator.next(); - double oldScore = hit.getRelevance().getScore(); - boolean didRerank = hitRescorer.rescoreHit(hit); - if (didRerank) { - ranges.withInitialScore(oldScore); - ranges.withFinalScore(hit.getRelevance().getScore()); - --rerankCount; - iterator.remove(); - } + int count = 0; + for (WrappedHit hit : hitsToRescore) { + if (count == rerankCount) break; + hitRescorer.preprocess(hit); + ++count; } + } + + private void runProcessing() { + int count = 0; + for (var iterator = hitsToRescore.iterator(); count < rerankCount && iterator.hasNext(); ) { + WrappedHit wrapped = iterator.next(); + double oldScore = wrapped.getScore(); + double newScore = hitRescorer.rescoreHit(wrapped); + ranges.withInitialScore(oldScore); + ranges.withFinalScore(newScore); + ++count; + iterator.remove(); + } + } + + private void runPostProcessing() { // if any hits are left in the list, they may need rescaling: - if (ranges.rescaleNeeded()) { + if (ranges.rescaleNeeded() && ! hitsToRescore.isEmpty()) { double scale = ranges.scale(); double bias = ranges.bias(); - for (Hit hit : hitsToRescore) { - double oldScore = hit.getRelevance().getScore(); - hit.setRelevance(oldScore * scale + bias); + for (WrappedHit wrapped : hitsToRescore) { + double oldScore = wrapped.getScore(); + wrapped.setScore(oldScore * scale + bias); } } - result.hits().sort(); } } diff --git a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java index a42024c80a1..548576e3a15 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java @@ -10,24 +10,23 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.Supplier; class SimpleEvaluator implements Evaluator { private final FunctionEvaluator evaluator; - private final Set<String> neededInputs; - - public SimpleEvaluator(FunctionEvaluator prototype) { - this.evaluator = prototype; - this.neededInputs = new HashSet<String>(prototype.function().arguments()); + + static Supplier<Evaluator> wrap(Supplier<FunctionEvaluator> supplier) { + return () -> new SimpleEvaluator(supplier.get()); } - @Override - public Collection<String> needInputs() { return List.copyOf(neededInputs); } + SimpleEvaluator(FunctionEvaluator prototype) { + this.evaluator = prototype; + } @Override - public SimpleEvaluator bind(String name, Tensor value) { - if (value != null) evaluator.bind(name, value); - neededInputs.remove(name); + public Evaluator bind(String name, Tensor value) { + evaluator.bind(name, value); return this; } @@ -42,7 +41,7 @@ class SimpleEvaluator implements Evaluator { buf.append("SimpleEvaluator("); buf.append(evaluator.function().toString()); buf.append(")["); - for (String arg : neededInputs) { + for (String arg : evaluator.function().arguments()) { buf.append("{").append(arg).append("}"); } buf.append("]"); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/WrappedHit.java b/container-search/src/main/java/com/yahoo/search/ranking/WrappedHit.java new file mode 100644 index 00000000000..7c33b836e33 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/WrappedHit.java @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; +import com.yahoo.tensor.Tensor; + +import static com.yahoo.searchlib.rankingexpression.Reference.RANKING_EXPRESSION_WRAPPER; + +import java.util.logging.Logger; + +class WrappedHit implements Comparable<WrappedHit> { + + private static final Logger logger = Logger.getLogger(WrappedHit.class.getName()); + private final Hit hit; + private final FeatureData matchFeatures; + private int idx = -1; + + private WrappedHit(Hit hit, FeatureData matchFeatures) { + this.hit = hit; + this.matchFeatures = matchFeatures; + } + + static WrappedHit from(Hit hit) { + if (hit.getField("matchfeatures") instanceof FeatureData mf) { + return new WrappedHit(hit, mf); + } else { + return null; + } + } + + double getScore() { + return hit.getRelevance().getScore(); + } + + void setScore(double value) { + hit.setRelevance(value); + } + + int getIdx() { + if (idx < 0) { + throw new IllegalStateException("Missing index"); + } + return idx; + } + + void setIdx(int value) { + if (idx == value) { + return; + } else if (idx < 0) { + idx = value; + } else { + throw new IllegalArgumentException("Cannot re-assign index " + idx + " -> " + value); + } + } + + public int compareTo(WrappedHit other) { + return hit.compareTo(other.hit); + } + + Tensor getTensor(String argName) { + var asTensor = matchFeatures.getTensor(argName); + if (asTensor == null) { + asTensor = matchFeatures.getTensor(alternate(argName)); + } + return asTensor; + } + + private static final String RE_PREFIX = RANKING_EXPRESSION_WRAPPER + "("; + private static final String RE_SUFFIX = ")"; + private static final int RE_PRE_LEN = RE_PREFIX.length(); + private static final int RE_SUF_LEN = RE_SUFFIX.length(); + + // rankingExpression(foo) <-> foo + static String alternate(String argName) { + if (argName.startsWith(RE_PREFIX) && argName.endsWith(RE_SUFFIX)) { + return argName.substring(RE_PRE_LEN, argName.length() - RE_SUF_LEN); + } else { + return RE_PREFIX + argName + RE_SUFFIX; + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java index 5fd41be0a54..fdd5c79adb6 100644 --- a/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java +++ b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java @@ -75,6 +75,7 @@ public class DefaultErrorHit extends Hit implements ErrorHit, Cloneable { /** Add all errors from another error hit to this */ public void addErrors(ErrorHit errorHit) { + if (this == errorHit) return; for (Iterator<? extends ErrorMessage> i = errorHit.errorIterator(); i.hasNext();) { addError(i.next()); } diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index c6447f85e61..51c0caf38a9 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -19,7 +19,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; /** * <p>A group of ordered hits. Since hitGroup is itself a kind of Hit, @@ -504,9 +503,7 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< int currentIndex = -1; ListenableArrayList<Hit> newHits = new ListenableArrayList<>(numHits); - for (Iterator<Hit> i = hits.iterator(); i.hasNext();) { - Hit hit = i.next(); - + for (Hit hit : hits) { if (hit.isAuxiliary()) { newHits.add(hit); } else { @@ -700,7 +697,6 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< // -------------- State bookkeeping /** Ensures result invariants. Must be called when a hit is added to this result. */ - @SuppressWarnings("deprecation") private void handleNewHit(Hit hit) { if (!hit.isAuxiliary()) concreteHitCount++; @@ -848,8 +844,8 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< HitGroup hitGroupClone = (HitGroup) super.clone(); hitGroupClone.hits = new ListenableArrayList<>(this.hits.size()); hitGroupClone.unmodifiableHits = Collections.unmodifiableList(hitGroupClone.hits); - for (Iterator<Hit> i = this.hits.iterator(); i.hasNext();) { - Hit hitClone = i.next().clone(); + for (Hit value : this.hits) { + Hit hitClone = value.clone(); hitGroupClone.hits.add(hitClone); } if (this.errorHit != null) { // Find the cloned error and assign it @@ -944,7 +940,7 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< } private Iterable<Hit> fillableHits() { - Predicate<Hit> isFillable = hit -> hit.isFillable(); + Predicate<Hit> isFillable = Hit::isFillable; return Iterables.filter(hits, isFillable); } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/FlatteningSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/FlatteningSearcherTestCase.java index bbd307c6fac..7ec35151eab 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/result/FlatteningSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/FlatteningSearcherTestCase.java @@ -12,6 +12,9 @@ import com.yahoo.search.Searcher; import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.grouping.request.GroupingOperation; import com.yahoo.search.grouping.vespa.GroupingExecutor; +import com.yahoo.search.result.DefaultErrorHit; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.searchchain.Execution; @@ -72,18 +75,21 @@ public class FlatteningSearcherTestCase { )); Execution execution = newExecution(new FlatteningSearcher(), new GroupingExecutor(ComponentId.fromString("grouping")), - new ResultProvider(Arrays.asList( + new ResultProvider(List.of( new GroupingListHit(List.of(group0), null), - new GroupingListHit(List.of(group1), null)))); + new GroupingListHit(List.of(group1), null))), + new HitsProvider(List.of( + new DefaultErrorHit("source 1", ErrorMessage.createBackendCommunicationError("backend communication error 1")), + new DefaultErrorHit("source 2", ErrorMessage.createBackendCommunicationError("backend communication error 1"))))); Result result = execution.search(query); - assertEquals(5, result.hits().size()); + assertEquals(6, result.hits().size()); assertFlat(result); assertEquals(2, result.getTotalHitCount()); } private void assertFlat(Result result) { for (var hit : result.hits()) - assertTrue(hit instanceof FastHit); + assertTrue(hit instanceof FastHit || hit instanceof ErrorHit); } private FS4Hit fs4Hit(double relevance) { @@ -126,5 +132,20 @@ public class FlatteningSearcherTestCase { return result; } } + @After (GroupingExecutor.COMPONENT_NAME) + private static class HitsProvider extends Searcher { + private final List<Hit> hits; + HitsProvider(List<Hit> hits) { + this.hits = hits; + } + @Override + public Result search(Query query, Execution exec) { + Result result = exec.search(query); + for (Hit hit : hits) { + result.hits().add(hit); + } + return result; + } + } } diff --git a/container-search/src/test/java/com/yahoo/search/ranking/NormalizerTestCase.java b/container-search/src/test/java/com/yahoo/search/ranking/NormalizerTestCase.java index 7373fb489f4..28d4cf67762 100644 --- a/container-search/src/test/java/com/yahoo/search/ranking/NormalizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/ranking/NormalizerTestCase.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.ranking; import org.junit.jupiter.api.Test; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java index 871f51689d5..c9df5a72a35 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java @@ -3,30 +3,58 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.config.provision.ClusterResources; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; +import com.yahoo.vespa.hosted.controller.api.integration.pricing.ApplicationResources; import com.yahoo.vespa.hosted.controller.api.integration.pricing.PriceInformation; import com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingController; import com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo; import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.List; +import static com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo.SupportLevel.BASIC; +import static java.math.BigDecimal.ZERO; +import static java.math.BigDecimal.valueOf; + public class MockPricingController implements PricingController { + // TODO: Remove when not in use anymore @Override public PriceInformation price(List<ClusterResources> clusterResources, PricingInfo pricingInfo, Plan plan) { - BigDecimal listPrice = BigDecimal.valueOf(clusterResources.stream() - .mapToDouble(resources -> resources.nodes() * - (resources.nodeResources().vcpu() * 1000 + - resources.nodeResources().memoryGb() * 100 + - resources.nodeResources().diskGb() * 10)) - .sum()) - .setScale(2, RoundingMode.HALF_UP); - BigDecimal volumeDiscount = new BigDecimal("-5.00"); - BigDecimal committedAmountDiscount = new BigDecimal("0.00"); - BigDecimal enclaveDiscount = new BigDecimal("0.00"); - BigDecimal totalAmount = listPrice.add(volumeDiscount); - return new PriceInformation(listPrice, volumeDiscount, committedAmountDiscount, enclaveDiscount, totalAmount); + BigDecimal listPrice = valueOf(clusterResources.stream() + .mapToDouble(resources -> resources.nodes() * + (resources.nodeResources().vcpu() * 1000 + + resources.nodeResources().memoryGb() * 100 + + resources.nodeResources().diskGb() * 10)) + .sum()); + + BigDecimal supportLevelCost = pricingInfo.supportLevel() == BASIC ? new BigDecimal("-160.00") : new BigDecimal("800.00"); + BigDecimal listPriceWithSupport = listPrice.add(supportLevelCost); + BigDecimal enclaveDiscount = pricingInfo.enclave() ? new BigDecimal("-15.1234") : BigDecimal.ZERO; + BigDecimal volumeDiscount = new BigDecimal("-5.64315634"); + BigDecimal committedAmountDiscount = new BigDecimal("-1.23"); + BigDecimal totalAmount = listPrice.add(supportLevelCost).add(enclaveDiscount).add(volumeDiscount).add(committedAmountDiscount); + return new PriceInformation(listPriceWithSupport, volumeDiscount, committedAmountDiscount, enclaveDiscount, totalAmount); + } + + @Override + public PriceInformation priceForApplications(List<ApplicationResources> applicationResources, PricingInfo pricingInfo, Plan plan) { + ApplicationResources resources = applicationResources.get(0); + BigDecimal listPrice = resources.vcpu().multiply(valueOf(1000)) + .add(resources.memoryGb().multiply(valueOf(100))) + .add(resources.diskGb().multiply(valueOf(10))) + .add(resources.enclaveVcpu().multiply(valueOf(1000)) + .add(resources.enclaveMemoryGb().multiply(valueOf(100))) + .add(resources.enclaveDiskGb().multiply(valueOf(10)))); + + BigDecimal supportLevelCost = pricingInfo.supportLevel() == BASIC ? new BigDecimal("-160.00") : new BigDecimal("800.00"); + BigDecimal listPriceWithSupport = listPrice.add(supportLevelCost); + BigDecimal enclaveDiscount = (resources.enclaveVcpu().compareTo(ZERO) > 0) ? new BigDecimal("-15.1234") : BigDecimal.ZERO; + BigDecimal volumeDiscount = new BigDecimal("-5.64315634"); + BigDecimal committedAmountDiscount = new BigDecimal("-1.23"); + BigDecimal totalAmount = listPrice.add(supportLevelCost).add(enclaveDiscount).add(volumeDiscount).add(committedAmountDiscount); + + return new PriceInformation(listPriceWithSupport, volumeDiscount, committedAmountDiscount, enclaveDiscount, totalAmount); } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java index 1bacbb6bcfe..1acb4964ea6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.billing; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; @@ -42,14 +43,22 @@ public class Bill { private final StatusHistory statusHistory; private final ZonedDateTime startTime; private final ZonedDateTime endTime; + private final String exportedId; - public Bill(Id id, TenantName tenant, StatusHistory statusHistory, List<LineItem> lineItems, ZonedDateTime startTime, ZonedDateTime endTime) { + public Bill(Id id, TenantName tenant, StatusHistory statusHistory, List<LineItem> lineItems, + ZonedDateTime startTime, ZonedDateTime endTime) { + this(id, tenant, statusHistory, lineItems, startTime, endTime, null); + } + + public Bill(Id id, TenantName tenant, StatusHistory statusHistory, List<LineItem> lineItems, + ZonedDateTime startTime, ZonedDateTime endTime, String exportedId) { this.id = id; this.tenant = tenant; this.lineItems = List.copyOf(lineItems); this.statusHistory = statusHistory; this.startTime = startTime; this.endTime = endTime; + this.exportedId = exportedId; } public Id id() { @@ -80,6 +89,10 @@ public class Bill { return endTime; } + public Optional<String> getExportedId() { + return Optional.ofNullable(exportedId); + } + public LocalDate getStartDate() { return startTime.toLocalDate(); } @@ -196,6 +209,8 @@ public class Bill { private BigDecimal gpuCost; private NodeResources.Architecture architecture; private int majorVersion; + private CloudAccount cloudAccount; + private String exportedId; public LineItem(String id, String description, BigDecimal amount, String plan, String agent, ZonedDateTime addedAt) { this.id = id; @@ -204,10 +219,15 @@ public class Bill { this.plan = plan; this.agent = agent; this.addedAt = addedAt; + this.cloudAccount = CloudAccount.empty; } - public LineItem(String id, String description, BigDecimal amount, String plan, String agent, ZonedDateTime addedAt, ZonedDateTime startedAt, ZonedDateTime endedAt, ApplicationId applicationId, ZoneId zoneId, - BigDecimal cpuHours, BigDecimal memoryHours, BigDecimal diskHours, BigDecimal gpuHours, BigDecimal cpuCost, BigDecimal memoryCost, BigDecimal diskCost, BigDecimal gpuCost, NodeResources.Architecture architecture, int majorVersion) { + public LineItem(String id, String description, BigDecimal amount, String plan, String agent, ZonedDateTime addedAt, + ZonedDateTime startedAt, ZonedDateTime endedAt, ApplicationId applicationId, ZoneId zoneId, + BigDecimal cpuHours, BigDecimal memoryHours, BigDecimal diskHours, BigDecimal gpuHours, BigDecimal cpuCost, + BigDecimal memoryCost, BigDecimal diskCost, BigDecimal gpuCost, NodeResources.Architecture architecture, + int majorVersion, CloudAccount cloudAccount, String exportedId) + { this(id, description, amount, plan, agent, addedAt); this.startedAt = startedAt; this.endedAt = endedAt; @@ -227,6 +247,8 @@ public class Bill { this.architecture = architecture; this.majorVersion = majorVersion; this.gpuCost = gpuCost; + this.cloudAccount = cloudAccount; + this.exportedId = exportedId; } /** The opaque ID of this */ @@ -319,6 +341,14 @@ public class Bill { return majorVersion; } + public CloudAccount getCloudAccount() { + return cloudAccount; + } + + public Optional<String> getExportedId() { + return Optional.ofNullable(exportedId); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java index c5c3bb894ca..3e24314ba5c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java @@ -144,4 +144,15 @@ public interface BillingDatabaseClient { * Performs necessary maintenance operations */ void maintain(); + + /** + * Set the invoice id from an external system for the given bill + */ + void setExportedInvoiceId(Bill.Id billId, String invoiceId); + + /** + * Set the invoice item id from an external system for the given line item + */ + void setExportedInvoiceItemId(String lineItemId, String invoiceItemId); + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java index 7ef85f0eaa6..300c1658c29 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java @@ -178,4 +178,11 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { @Override public void maintain() {} + + @Override + public void setExportedInvoiceId(Bill.Id billId, String invoiceId) { } + + @Override + public void setExportedInvoiceItemId(String lineItemId, String invoiceItemId) { } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java index 452143da75d..3ae2b0aa495 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java @@ -135,7 +135,8 @@ public class PlanRegistryMock implements PlanRegistry { dgbCost, gpuCost, usage.getArchitecture(), - usage.getMajorVersion() + usage.getMajorVersion(), + usage.getCloudAccount() ); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificate.java index 1a6974566d4..09120f8cd21 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificate.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificate.java @@ -135,4 +135,27 @@ public record EndpointCertificate(String keyName, String certName, int version, this.generatedId); } + /** Returns whether given DNS name matches any of the requested SANs in this */ + public boolean sanMatches(String dnsName) { + return sanMatches(dnsName, requestedDnsSans); + } + + static boolean sanMatches(String dnsName, List<String> sanDnsNames) { + return sanDnsNames.stream().anyMatch(sanDnsName -> sanMatches(dnsName, sanDnsName)); + } + + private static boolean sanMatches(String dnsName, String sanDnsName) { + String[] sanNameParts = sanDnsName.split("\\."); + String[] dnsNameParts = dnsName.split("\\."); + if (sanNameParts.length != dnsNameParts.length || sanNameParts.length == 0) { + return false; + } + for (int i = 0; i < sanNameParts.length; i++) { + if (!sanNameParts[i].equals("*") && !sanNameParts[i].equals(dnsNameParts[i])) { + return false; + } + } + return true; + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/ApplicationResources.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/ApplicationResources.java new file mode 100644 index 00000000000..5c6de406a55 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/ApplicationResources.java @@ -0,0 +1,21 @@ +package com.yahoo.vespa.hosted.controller.api.integration.pricing; + +import java.math.BigDecimal; + +/** + * @param applicationName name of the application + * @param vcpu vcpus summed over all clusters, instances, zones + * @param memoryGb memory in Gb summed over all clusters, instances, zones + * @param diskGb disk in Gb summed over all clusters, instances, zones + * @param gpuMemoryGb GPU memory in Gb summed over all clusters, instances, zones + * @param enclaveVcpu vcpus summed over all clusters, instances, zones + * @param enclaveMemoryGb memory in Gb summed over all clusters, instances, zones + * @param enclaveDiskGb disk in Gb summed over all clusters, instances, zones + * @param enclaveGpuMemoryGb GPU memory in Gb summed over all clusters, instances, zones + */ +public record ApplicationResources(String applicationName, BigDecimal vcpu, BigDecimal memoryGb, BigDecimal diskGb, + BigDecimal gpuMemoryGb, BigDecimal enclaveVcpu, BigDecimal enclaveMemoryGb, + BigDecimal enclaveDiskGb, BigDecimal enclaveGpuMemoryGb) { + +} + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PriceInformation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PriceInformation.java index 2a6ecf87180..887741f9196 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PriceInformation.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PriceInformation.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.pricing; import java.math.BigDecimal; -public record PriceInformation(BigDecimal listPrice, BigDecimal volumeDiscount, BigDecimal committedAmountDiscount, +public record PriceInformation(BigDecimal listPriceWithSupport, BigDecimal volumeDiscount, BigDecimal committedAmountDiscount, BigDecimal enclaveDiscount, BigDecimal totalAmount) { } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PriceInformationApplications.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PriceInformationApplications.java new file mode 100644 index 00000000000..d7da2308a16 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PriceInformationApplications.java @@ -0,0 +1,9 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.pricing; + +import java.math.BigDecimal; + +public record PriceInformationApplications(BigDecimal listPriceWithSupport, BigDecimal volumeDiscount, BigDecimal committedAmountDiscount, + BigDecimal enclaveDiscount, BigDecimal totalAmount) { + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PricingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PricingController.java index d8186f17796..85d83a32e4c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PricingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PricingController.java @@ -13,6 +13,16 @@ import java.util.List; */ public interface PricingController { + // TOD: Legacy, will be removed when not in use anymore PriceInformation price(List<ClusterResources> clusterResources, PricingInfo pricingInfo, Plan plan); + /** + * + * @param applicationResources resources used by an application + * @param pricingInfo pricing info + * @param plan the plan to use for this calculation + * @return a PriceInformation instance + */ + PriceInformation priceForApplications(List<ApplicationResources> applicationResources, PricingInfo pricingInfo, Plan plan); + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostInfo.java index 0ffe3768e71..0dcfb2d9823 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostInfo.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/CostInfo.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.resource; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; @@ -24,11 +25,14 @@ public class CostInfo { private final BigDecimal gpuCost; private final NodeResources.Architecture architecture; private final int majorVersion; + private final CloudAccount cloudAccount; public CostInfo(ApplicationId applicationId, ZoneId zoneId, BigDecimal cpuHours, BigDecimal memoryHours, BigDecimal diskHours, BigDecimal gpuHours, - BigDecimal cpuCost, BigDecimal memoryCost, BigDecimal diskCost, BigDecimal gpuCost, NodeResources.Architecture architecture, int majorVersion) { + BigDecimal cpuCost, BigDecimal memoryCost, BigDecimal diskCost, BigDecimal gpuCost, NodeResources.Architecture architecture, + int majorVersion, CloudAccount cloudAccount) + { this.applicationId = applicationId; this.zoneId = zoneId; this.cpuHours = cpuHours; @@ -41,6 +45,7 @@ public class CostInfo { this.gpuCost = gpuCost; this.architecture = architecture; this.majorVersion = majorVersion; + this.cloudAccount = cloudAccount; } public ApplicationId getApplicationId() { @@ -95,4 +100,7 @@ public class CostInfo { return majorVersion; } + public CloudAccount getCloudAccount() { + return cloudAccount; + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java index 4879e92c779..81dfdf4656c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceDatabaseClientMock.java @@ -80,6 +80,7 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { plan, a.resources().architecture(), a.getMajorVersion(), + a.getAccount(), BigDecimal.valueOf(a.resources().vcpu()).multiply(d), BigDecimal.valueOf(a.resources().memoryGb()).multiply(d), BigDecimal.valueOf(a.resources().diskGb()).multiply(d), @@ -93,12 +94,14 @@ public class ResourceDatabaseClientMock implements ResourceDatabaseClient { assert a.getZoneId().equals(b.getZoneId()); assert a.getPlan().equals(b.getPlan()); assert a.getArchitecture().equals(b.getArchitecture()); + assert a.getCloudAccount().equals(b.getCloudAccount()); return new ResourceUsage( a.getApplicationId(), a.getZoneId(), a.getPlan(), a.getArchitecture(), a.getMajorVersion(), + a.getCloudAccount(), a.getCpuMillis().add(b.getCpuMillis()), a.getMemoryMillis().add(b.getMemoryMillis()), a.getDiskMillis().add(b.getDiskMillis()), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java index b433666b194..dc14a043183 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceSnapshot.java @@ -1,16 +1,16 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.resource; -import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import java.time.Instant; +import java.util.Collection; import java.util.List; import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; /** @@ -32,37 +32,40 @@ public class ResourceSnapshot { private final Instant timestamp; private final ZoneId zoneId; private final int majorVersion; + private final CloudAccount account; - public ResourceSnapshot(ApplicationId applicationId, NodeResources resources, Instant timestamp, ZoneId zoneId, int majorVersion) { + public ResourceSnapshot(ApplicationId applicationId, NodeResources resources, Instant timestamp, ZoneId zoneId, int majorVersion, CloudAccount account) { this.applicationId = applicationId; this.resources = resources; this.timestamp = timestamp; this.zoneId = zoneId; this.majorVersion = majorVersion; + this.account = account; } public static ResourceSnapshot from(ApplicationId applicationId, int nodes, NodeResources resources, Instant timestamp, ZoneId zoneId) { - return new ResourceSnapshot(applicationId, resources.multipliedBy(nodes), timestamp, zoneId, 0); + return new ResourceSnapshot(applicationId, resources.multipliedBy(nodes), timestamp, zoneId, 0, CloudAccount.empty); } public static ResourceSnapshot from(List<Node> nodes, Instant timestamp, ZoneId zoneId) { - Set<ApplicationId> applicationIds = nodes.stream() - .filter(node -> node.owner().isPresent()) - .map(node -> node.owner().get()) - .collect(Collectors.toSet()); + var application = exactlyOne("application", nodes.stream() + .filter(node -> node.owner().isPresent()) + .map(node -> node.owner().get()) + .collect(Collectors.toSet())); - Set<Integer> versions = nodes.stream() + var version = exactlyOne("version", nodes.stream() .map(n -> n.wantedVersion().getMajor()) - .collect(Collectors.toSet()); + .collect(Collectors.toSet())); - if (applicationIds.size() != 1) throw new IllegalArgumentException("List of nodes can only represent one application"); - if (versions.size() != 1) throw new IllegalArgumentException("List of nodes can only represent one version"); + var account = exactlyOne("account", nodes.stream() + .map(Node::cloudAccount) + .collect(Collectors.toSet())); var resources = nodes.stream() .map(Node::resources) .reduce(zero, ResourceSnapshot::addResources); - return new ResourceSnapshot(applicationIds.iterator().next(), resources, timestamp, zoneId, versions.iterator().next()); + return new ResourceSnapshot(application, resources, timestamp, zoneId, version, account); } public ApplicationId getApplicationId() { @@ -85,6 +88,10 @@ public class ResourceSnapshot { return majorVersion; } + public CloudAccount getAccount() { + return account; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -122,4 +129,9 @@ public class ResourceSnapshot { a.architecture() == NodeResources.Architecture.any ? b.architecture() : a.architecture(), a.gpuResources().plus(b.gpuResources())); } + + private static <T> T exactlyOne(String resource, Collection<T> collection) { + if (collection.size() != 1) throw new IllegalArgumentException("More than one '" + resource + "', was: " + collection.size()); + return collection.iterator().next(); + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java index b3ce0fffada..3cb611af8a0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/ResourceUsage.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.resource; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; @@ -20,9 +21,11 @@ public class ResourceUsage { private final BigDecimal gpuMillis; private final NodeResources.Architecture architecture; private final int majorVersion; + private final CloudAccount cloudAccount; public ResourceUsage(ApplicationId applicationId, ZoneId zoneId, Plan plan, NodeResources.Architecture architecture, - int majorVersion, BigDecimal cpuMillis, BigDecimal memoryMillis, BigDecimal diskMillis, BigDecimal gpuMillis) { + int majorVersion, CloudAccount cloudAccount, BigDecimal cpuMillis, BigDecimal memoryMillis, BigDecimal diskMillis, BigDecimal gpuMillis) + { this.applicationId = applicationId; this.zoneId = zoneId; this.cpuMillis = cpuMillis; @@ -32,6 +35,7 @@ public class ResourceUsage { this.plan = plan; this.architecture = architecture; this.majorVersion = majorVersion; + this.cloudAccount = cloudAccount; } public ApplicationId getApplicationId() { @@ -67,4 +71,8 @@ public class ResourceUsage { public NodeResources.Architecture getArchitecture() { return architecture; } + + public CloudAccount getCloudAccount(){ + return cloudAccount; + } } diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateTest.java new file mode 100644 index 00000000000..e165157dac2 --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateTest.java @@ -0,0 +1,31 @@ +package com.yahoo.vespa.hosted.controller.api.integration.certificates; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author mpolden + */ +class EndpointCertificateTest { + + @Test + public void san_matches() { + List<String> sans = List.of("*.a.example.com", "b.example.com", "c.example.com"); + assertTrue(EndpointCertificate.sanMatches("b.example.com", sans)); + assertTrue(EndpointCertificate.sanMatches("c.example.com", sans)); + assertTrue(EndpointCertificate.sanMatches("foo.a.example.com", sans)); + assertFalse(EndpointCertificate.sanMatches("", List.of())); + assertFalse(EndpointCertificate.sanMatches("example.com", List.of())); + assertFalse(EndpointCertificate.sanMatches("example.com", sans)); + assertFalse(EndpointCertificate.sanMatches("d.example.com", sans)); + assertFalse(EndpointCertificate.sanMatches("a.example.com", sans)); + assertFalse(EndpointCertificate.sanMatches("aa.example.com", sans)); + assertFalse(EndpointCertificate.sanMatches("c.c.example.com", sans)); + assertFalse(EndpointCertificate.sanMatches("a.a.a.example.com", sans)); + } + +} diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 6671b71c73f..a9db2cace85 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -118,6 +118,33 @@ <!-- compile --> <dependency> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity-engine-core</artifactId> + <exclusions> + <exclusion> + <!-- Must use the one provided by Jdisc to prevent two instances of slf4j classes. --> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.velocity.tools</groupId> + <artifactId>velocity-tools-generic</artifactId> + <exclusions> + <exclusion> + <!-- Must use the one provided by Jdisc to prevent two instances of slf4j classes. --> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-csv</artifactId> </dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 50718429a2b..0de0ea06904 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -524,7 +524,7 @@ public class ApplicationController { try (Mutex lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); - return prepareEndpoints(deployment, job, application, applicationPackage, deployLogger); + return prepareEndpoints(deployment, job, application, applicationPackage, deployLogger, lock); } }; @@ -569,13 +569,16 @@ public class ApplicationController { private PreparedEndpoints prepareEndpoints(DeploymentId deployment, JobId job, LockedApplication application, ApplicationPackageStream applicationPackage, - Consumer<String> deployLogger) { + Consumer<String> deployLogger, + Mutex applicationLock) { Instance instance = application.get().require(job.application().instance()); Tags tags = applicationPackage.truncatedPackage().deploymentSpec().instance(instance.name()) .map(DeploymentInstanceSpec::tags) .orElseGet(Tags::empty); - Optional<EndpointCertificate> certificate = endpointCertificates.get(instance, deployment.zoneId(), applicationPackage.truncatedPackage().deploymentSpec()); - certificate.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); + EndpointCertificate certificate = endpointCertificates.get(deployment, + applicationPackage.truncatedPackage().deploymentSpec(), + applicationLock); + deployLogger.accept("Using CA signed certificate version %s".formatted(certificate.version())); BasicServicesXml services = applicationPackage.truncatedPackage().services(deployment, tags); return controller.routing().of(deployment).prepare(services, certificate, application); } @@ -696,7 +699,7 @@ public class ApplicationController { if (preparedEndpoints == null) return DeploymentEndpoints.none; PreparedEndpoints prepared = preparedEndpoints.get(); generatedEndpoints.set(prepared.endpoints().generated()); - return new DeploymentEndpoints(prepared.containerEndpoints(), prepared.certificate()); + return new DeploymentEndpoints(prepared.containerEndpoints(), Optional.of(prepared.certificate())); }; Supplier<List<DataplaneTokenVersions>> dataplaneTokenVersions = () -> { Tags tags = applicationPackage.truncatedPackage().deploymentSpec() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 98d8feda0bb..51e20d0017c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import com.yahoo.vespa.hosted.controller.routing.GeneratedEndpointList; import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints; import com.yahoo.vespa.hosted.controller.routing.RoutingId; @@ -121,8 +122,21 @@ public class RoutingController { return rotationRepository; } + /** Returns the endpoint config to use for given instance */ + public EndpointConfig endpointConfig(ApplicationId instance) { + // TODO(mpolden): Switch to reading endpoint-config flag + if (legacyEndpointsEnabled(instance)) { + if (generatedEndpointsEnabled(instance)) { + return EndpointConfig.combined; + } else { + return EndpointConfig.legacy; + } + } + return EndpointConfig.generated; + } + /** Prepares and returns the endpoints relevant for given deployment */ - public PreparedEndpoints prepare(DeploymentId deployment, BasicServicesXml services, Optional<EndpointCertificate> certificate, LockedApplication application) { + public PreparedEndpoints prepare(DeploymentId deployment, BasicServicesXml services, EndpointCertificate certificate, LockedApplication application) { EndpointList endpoints = EndpointList.EMPTY; DeploymentSpec spec = application.get().deploymentSpec(); @@ -136,7 +150,7 @@ public class RoutingController { // Add zone-scoped endpoints Map<EndpointId, List<GeneratedEndpoint>> generatedForDeclaredEndpoints = new HashMap<>(); Set<ClusterSpec.Id> clustersWithToken = new HashSet<>(); - boolean generatedEndpointsEnabled = generatedEndpointsEnabled(deployment.applicationId()); + EndpointConfig config = endpointConfig(deployment.applicationId()); RoutingPolicyList applicationPolicies = policies().read(TenantAndApplicationId.from(deployment.applicationId())); RoutingPolicyList deploymentPolicies = applicationPolicies.deployment(deployment); for (var container : services.containers()) { @@ -153,7 +167,7 @@ public class RoutingController { if (tokenSupported) { generatedForCluster = generateEndpoints(AuthMethod.token, certificate, Optional.empty(), generatedForCluster); } - GeneratedEndpointList generatedEndpoints = generatedEndpointsEnabled ? GeneratedEndpointList.copyOf(generatedForCluster) : GeneratedEndpointList.EMPTY; + GeneratedEndpointList generatedEndpoints = config.supportsGenerated() ? GeneratedEndpointList.copyOf(generatedForCluster) : GeneratedEndpointList.EMPTY; endpoints = endpoints.and(endpointsOf(deployment, clusterId, generatedEndpoints).scope(Scope.zone)); } @@ -185,7 +199,7 @@ public class RoutingController { return generatedEndpoints; }); }); - Map<EndpointId, GeneratedEndpointList> generatedEndpoints = generatedEndpointsEnabled + Map<EndpointId, GeneratedEndpointList> generatedEndpoints = config.supportsGenerated() ? generatedForDeclaredEndpoints.entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, kv -> GeneratedEndpointList.copyOf(kv.getValue()))) @@ -380,7 +394,24 @@ public class RoutingController { } /** Returns certificate DNS names (CN and SAN values) for given deployment */ - public List<String> certificateDnsNames(DeploymentId deployment, DeploymentSpec deploymentSpec) { + public List<String> certificateDnsNames(DeploymentId deployment, DeploymentSpec deploymentSpec, String generatedId, boolean legacy) { + List<String> endpointDnsNames = new ArrayList<>(); + if (legacy) { + endpointDnsNames.addAll(legacyCertificateDnsNames(deployment, deploymentSpec)); + } + for (Scope scope : List.of(Scope.zone, Scope.global, Scope.application)) { + endpointDnsNames.add(Endpoint.of(deployment.applicationId()) + .wildcardGenerated(generatedId, scope) + .routingMethod(RoutingMethod.exclusive) + .on(Port.tls()) + .certificateName() + .in(controller.system()) + .dnsName()); + } + return Collections.unmodifiableList(endpointDnsNames); + } + + private List<String> legacyCertificateDnsNames(DeploymentId deployment, DeploymentSpec deploymentSpec) { List<String> endpointDnsNames = new ArrayList<>(); // We add first an endpoint name based on a hash of the application ID, @@ -447,10 +478,7 @@ public class RoutingController { } private EndpointList filterEndpoints(ApplicationId instance, EndpointList endpoints) { - if (generatedEndpointsEnabled(instance) && !legacyEndpointsEnabled(instance)) { - return endpoints.generated(); - } - return endpoints; + return endpointConfig(instance) == EndpointConfig.generated ? endpoints.generated() : endpoints; } private void registerRotationEndpointsInDns(PreparedEndpoints prepared) { @@ -491,13 +519,13 @@ public class RoutingController { } /** Returns generated endpoints. A new endpoint is generated if no matching endpoint already exists */ - private List<GeneratedEndpoint> generateEndpoints(AuthMethod authMethod, Optional<EndpointCertificate> certificate, + private List<GeneratedEndpoint> generateEndpoints(AuthMethod authMethod, EndpointCertificate certificate, Optional<EndpointId> declaredEndpoint, List<GeneratedEndpoint> current) { if (current.stream().anyMatch(e -> e.authMethod() == authMethod && e.endpoint().equals(declaredEndpoint))) { return current; } - Optional<String> applicationPart = certificate.flatMap(EndpointCertificate::generatedId); + Optional<String> applicationPart = certificate.generatedId(); if (applicationPart.isPresent()) { current = new ArrayList<>(current); current.add(new GeneratedEndpoint(GeneratedEndpoint.createPart(controller.random(true)), @@ -572,14 +600,14 @@ public class RoutingController { return Collections.unmodifiableList(routingMethods); } - public boolean generatedEndpointsEnabled(ApplicationId instance) { + private boolean generatedEndpointsEnabled(ApplicationId instance) { return generatedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) .value(); } - public boolean legacyEndpointsEnabled(ApplicationId instance) { + private boolean legacyEndpointsEnabled(ApplicationId instance) { return legacyEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index d95bb0f9f1b..39e1c89c202 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -65,7 +65,7 @@ public class Endpoint { Objects.requireNonNull(generated, "generated must be non-null"); this.id = requireEndpointId(id, scope, certificateName); this.cluster = requireCluster(cluster, certificateName); - this.instance = requireInstance(instanceName, scope); + this.instance = requireInstance(instanceName, scope, certificateName, generated.isPresent()); this.url = url; this.targets = List.copyOf(requireTargets(targets, application, instanceName, scope, certificateName)); this.scope = requireScope(scope, routingMethod); @@ -259,7 +259,7 @@ public class Endpoint { } /** Returns the DNS suffix used for endpoints in given system */ - public static String dnsSuffix(SystemName system) { + private static String dnsSuffix(SystemName system) { return switch (system) { case cd -> CD_OATH_DNS_SUFFIX; case main -> MAIN_OATH_DNS_SUFFIX; @@ -316,7 +316,10 @@ public class Endpoint { return endpointId; } - private static Optional<InstanceName> requireInstance(Optional<InstanceName> instanceName, Scope scope) { + private static Optional<InstanceName> requireInstance(Optional<InstanceName> instanceName, Scope scope, boolean certificateName, boolean generated) { + if (generated && certificateName) { + return instanceName; + } if (scope == Scope.application) { if (instanceName.isPresent()) throw new IllegalArgumentException("Instance cannot be set for scope " + scope); } else { @@ -331,7 +334,8 @@ public class Endpoint { } private static List<Target> requireTargets(List<Target> targets, TenantAndApplicationId application, Optional<InstanceName> instanceName, Scope scope, boolean certificateName) { - if (!certificateName && targets.isEmpty()) throw new IllegalArgumentException("At least one target must be given for " + scope + " endpoints"); + if (certificateName && targets.isEmpty()) return List.of(); + if (targets.isEmpty()) throw new IllegalArgumentException("At least one target must be given for " + scope + " endpoints"); if (scope == Scope.zone && targets.size() != 1) throw new IllegalArgumentException("Exactly one target must be given for " + scope + " endpoints"); for (var target : targets) { if (scope == Scope.application) { @@ -524,6 +528,18 @@ public class Endpoint { return target(ClusterSpec.Id.from("*"), deployment); } + /** Sets the generated wildcard target for this */ + public EndpointBuilder wildcardGenerated(String applicationPart, Scope scope) { + this.cluster = ClusterSpec.Id.from("*"); + if (scope.multiDeployment()) { + this.endpointId = EndpointId.of("*"); + } + this.targets = List.of(); + this.scope = requireUnset(scope); + this.generated = Optional.of(new GeneratedEndpoint("*", applicationPart, AuthMethod.mtls, Optional.ofNullable(endpointId))); + return this; + } + /** Sets the application target with given ID, cluster, deployments and their weights */ public EndpointBuilder targetApplication(EndpointId endpointId, ClusterSpec.Id cluster, Map<DeploymentId, Integer> deployments) { this.endpointId = endpointId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java index b7929240d76..5f75d6105b5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java @@ -17,7 +17,8 @@ import java.util.regex.Pattern; */ public record GeneratedEndpoint(String clusterPart, String applicationPart, AuthMethod authMethod, Optional<EndpointId> endpoint) { - private static final Pattern PART_PATTERN = Pattern.compile("^[a-f][a-f0-9]{7}$"); + private static final Pattern CLUSTER_PART_PATTERN = Pattern.compile("^([a-f][a-f0-9]{7}|\\*)$"); + private static final Pattern APPLICATION_PART_PATTERN = Pattern.compile("^[a-f][a-f0-9]{7}$"); public GeneratedEndpoint { Objects.requireNonNull(clusterPart); @@ -25,8 +26,8 @@ public record GeneratedEndpoint(String clusterPart, String applicationPart, Auth Objects.requireNonNull(authMethod); Objects.requireNonNull(endpoint); - Validation.requireMatch(clusterPart, "Cluster part", PART_PATTERN); - Validation.requireMatch(applicationPart, "Application part", PART_PATTERN); + Validation.requireMatch(clusterPart, "Cluster part", CLUSTER_PART_PATTERN); + Validation.requireMatch(applicationPart, "Application part", APPLICATION_PART_PATTERN); } /** Returns whether this was generated for an endpoint declared in {@link com.yahoo.config.application.api.DeploymentSpec} */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java index 18b537efd8c..49e2dc5bb0d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java @@ -15,10 +15,19 @@ import java.util.Optional; */ public record AssignedCertificate(TenantAndApplicationId application, Optional<InstanceName> instance, - EndpointCertificate certificate) { + EndpointCertificate certificate, + boolean shouldValidate) { public AssignedCertificate with(EndpointCertificate certificate) { - return new AssignedCertificate(application, instance, certificate); + return new AssignedCertificate(application, instance, certificate, shouldValidate); + } + + public AssignedCertificate withoutInstance() { + return new AssignedCertificate(application, Optional.empty(), certificate, shouldValidate); + } + + public AssignedCertificate withShouldValidate(boolean shouldValidate) { + return new AssignedCertificate(application, instance, certificate, shouldValidate); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index 9f03e3f0072..391c9806f0a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -11,18 +11,18 @@ import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateValidator; import com.yahoo.vespa.hosted.controller.api.integration.secrets.GcpSecretStore; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import java.time.Clock; import java.time.Duration; @@ -30,26 +30,29 @@ import java.time.Instant; import java.util.Comparator; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate.State; /** - * Looks up stored endpoint certificate, provisions new certificates if none is found, - * and re-provisions the certificate if the deploying-to zone is not covered. + * This provisions, assigns and updates the certificate for a given deployment. * * See also {@link com.yahoo.vespa.hosted.controller.maintenance.EndpointCertificateMaintainer}, which handles * refreshes, deletions and triggers deployments. * * @author andreer + * @author mpolden */ public class EndpointCertificates { - private static final Logger log = Logger.getLogger(EndpointCertificates.class.getName()); + private static final Logger LOG = Logger.getLogger(EndpointCertificates.class.getName()); + private static final Duration GCP_CERTIFICATE_EXPIRY_TIME = Duration.ofDays(100); // 100 days, 10 more than notAfter time private final Controller controller; private final CuratorDb curator; @@ -58,150 +61,216 @@ public class EndpointCertificates { private final EndpointCertificateValidator certificateValidator; private final BooleanFlag useAlternateCertProvider; private final StringFlag endpointCertificateAlgo; - private final BooleanFlag assignLegacyNames; - private final static Duration GCP_CERTIFICATE_EXPIRY_TIME = Duration.ofDays(100); // 100 days, 10 more than notAfter time public EndpointCertificates(Controller controller, EndpointCertificateProvider certificateProvider, EndpointCertificateValidator certificateValidator) { this.controller = controller; this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); - this.assignLegacyNames = Flags.LEGACY_ENDPOINTS.bindTo(controller.flagSource()); this.curator = controller.curator(); this.clock = controller.clock(); this.certificateProvider = certificateProvider; this.certificateValidator = certificateValidator; } - /** Returns a suitable certificate for endpoints of given instance and zone */ - public Optional<EndpointCertificate> get(Instance instance, ZoneId zone, DeploymentSpec deploymentSpec) { + /** Returns a suitable certificate for endpoints of given deployment */ + public EndpointCertificate get(DeploymentId deployment, DeploymentSpec deploymentSpec, Mutex applicationLock) { + Objects.requireNonNull(applicationLock); Instant start = clock.instant(); - Optional<EndpointCertificate> cert = getOrProvision(instance, zone, deploymentSpec); + EndpointConfig config = controller.routing().endpointConfig(deployment.applicationId()); + EndpointCertificate certificate = assignTo(deployment, deploymentSpec, config); Duration duration = Duration.between(start, clock.instant()); - if (duration.toSeconds() > 30) - log.log(Level.INFO, Text.format("Getting endpoint certificate for %s took %d seconds!", instance.id().serializedForm(), duration.toSeconds())); - - if (controller.zoneRegistry().zones().all().in(CloudName.GCP).ids().contains(zone)) { // Until CKMS is available from GCP - if (cert.isPresent()) { - // Validate before copying cert to GCP. This will ensure we don't bug out on the first deployment, but will take more time - certificateValidator.validate(cert.get(), instance.id().serializedForm(), zone, controller.routing().certificateDnsNames(new DeploymentId(instance.id(), zone), deploymentSpec)); - GcpSecretStore gcpSecretStore = controller.serviceRegistry().gcpSecretStore(); - String mangledCertName = "endpointCert_" + cert.get().certName().replace('.', '_') + "-v" + cert.get().version(); // Google cloud does not accept dots in secrets, but they accept underscores - String mangledKeyName = "endpointCert_" + cert.get().keyName().replace('.', '_') + "-v" + cert.get().version(); // Google cloud does not accept dots in secrets, but they accept underscores - if (gcpSecretStore.getLatestSecretVersion(mangledCertName) == null) { - gcpSecretStore.setSecret(mangledCertName, - Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), - "endpoint-cert-accessor"); - gcpSecretStore.addSecretVersion(mangledCertName, - controller.secretStore().getSecret(cert.get().certName(), cert.get().version())); - } - if (gcpSecretStore.getLatestSecretVersion(mangledKeyName) == null) { - gcpSecretStore.setSecret(mangledKeyName, - Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), - "endpoint-cert-accessor"); - gcpSecretStore.addSecretVersion(mangledKeyName, - controller.secretStore().getSecret(cert.get().keyName(), cert.get().version())); - } - - return Optional.of(cert.get().withVersion(1).withKeyName(mangledKeyName).withCertName(mangledCertName)); - } + if (duration.toSeconds() > 30) { + LOG.log(Level.INFO, Text.format("Getting endpoint certificate for %s took %d seconds!", deployment.applicationId().serializedForm(), duration.toSeconds())); } + if (isGcp(deployment)) { + // This is needed until CKMS is available from GCP + return validateGcpCertificate(deployment, deploymentSpec, certificate, config); + } + return certificate; + } - return cert; + private boolean isGcp(DeploymentId deployment) { + return controller.zoneRegistry().zones().all().in(CloudName.GCP).ids().contains(deployment.zoneId()); } - private EndpointCertificate assignFromPool(Instance instance, ZoneId zone) { - // For deployments to manually deployed environments: use per instance certificate - // For all other environments (apply in order): - // * Use per instance certificate if it exists and is assigned a randomized id - // * Use per application certificate if it exits and is assigned a randomized id - // * Assign from pool - - TenantAndApplicationId application = TenantAndApplicationId.from(instance.id()); - Optional<AssignedCertificate> perInstanceAssignedCertificate = curator.readAssignedCertificate(application, Optional.of(instance.name())); - if (perInstanceAssignedCertificate.isPresent() && perInstanceAssignedCertificate.get().certificate().generatedId().isPresent()) { - return updateLastRequested(perInstanceAssignedCertificate.get()).certificate(); - } else if (! zone.environment().isManuallyDeployed()) { - Optional<AssignedCertificate> perApplicationAssignedCertificate = curator.readAssignedCertificate(application, Optional.empty()); - if (perApplicationAssignedCertificate.isPresent() && perApplicationAssignedCertificate.get().certificate().generatedId().isPresent()) { - return updateLastRequested(perApplicationAssignedCertificate.get()).certificate(); - } + private EndpointCertificate validateGcpCertificate(DeploymentId deployment, DeploymentSpec deploymentSpec, EndpointCertificate certificate, EndpointConfig config) { + // Validate before copying cert to GCP. This will ensure we don't bug out on the first deployment, but will take more time + List<String> dnsNames = controller.routing().certificateDnsNames(deployment, deploymentSpec, certificate.generatedId().get(), config.supportsLegacy()); + certificateValidator.validate(certificate, deployment.applicationId().serializedForm(), deployment.zoneId(), dnsNames); + GcpSecretStore gcpSecretStore = controller.serviceRegistry().gcpSecretStore(); + String mangledCertName = "endpointCert_" + certificate.certName().replace('.', '_') + "-v" + certificate.version(); // Google cloud does not accept dots in secrets, but they accept underscores + String mangledKeyName = "endpointCert_" + certificate.keyName().replace('.', '_') + "-v" + certificate.version(); // Google cloud does not accept dots in secrets, but they accept underscores + if (gcpSecretStore.getLatestSecretVersion(mangledCertName) == null) { + gcpSecretStore.setSecret(mangledCertName, + Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), + "endpoint-cert-accessor"); + gcpSecretStore.addSecretVersion(mangledCertName, + controller.secretStore().getSecret(certificate.certName(), certificate.version())); + } + if (gcpSecretStore.getLatestSecretVersion(mangledKeyName) == null) { + gcpSecretStore.setSecret(mangledKeyName, + Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), + "endpoint-cert-accessor"); + gcpSecretStore.addSecretVersion(mangledKeyName, + controller.secretStore().getSecret(certificate.keyName(), certificate.version())); } + return certificate.withVersion(1).withKeyName(mangledKeyName).withCertName(mangledCertName); + } - // For new applications which is assigned from pool we follow these rules: - // Assign certificate per instance only in manually deployed environments. In other environments, we share the - // certificate because application endpoints can span instances - Optional<InstanceName> instanceName = zone.environment().isManuallyDeployed() ? Optional.of(instance.name()) : Optional.empty(); + private AssignedCertificate assignFromPool(TenantAndApplicationId application, Optional<InstanceName> instanceName, ZoneId zone) { try (Mutex lock = controller.curator().lockCertificatePool()) { Optional<UnassignedCertificate> candidate = curator.readUnassignedCertificates().stream() .filter(pc -> pc.state() == State.ready) .min(Comparator.comparingLong(pc -> pc.certificate().lastRequested())); if (candidate.isEmpty()) { - throw new IllegalArgumentException("No endpoint certificate available in pool, for deployment of " + instance.id() + " in " + zone); + throw new IllegalArgumentException("No endpoint certificate available in pool, for deployment of " + + application + instanceName.map(i -> "." + i.value()).orElse("") + + " in " + zone); } try (NestedTransaction transaction = new NestedTransaction()) { curator.removeUnassignedCertificate(candidate.get(), transaction); - EndpointCertificate certificate = candidate.get().certificate().withLastRequested(clock.instant().getEpochSecond()); - curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, certificate), - transaction); + AssignedCertificate assigned = new AssignedCertificate(application, instanceName, candidate.get().certificate(), false); + curator.writeAssignedCertificate(assigned, transaction); transaction.commit(); - return certificate; + return assigned; } } } - AssignedCertificate updateLastRequested(AssignedCertificate assignedCertificate) { - AssignedCertificate updated = assignedCertificate.with(assignedCertificate.certificate().withLastRequested(clock.instant().getEpochSecond())); - curator.writeAssignedCertificate(updated); - return updated; + private AssignedCertificate instanceLevelCertificate(DeploymentId deployment, DeploymentSpec deploymentSpec, boolean allowPool) { + TenantAndApplicationId application = TenantAndApplicationId.from(deployment.applicationId()); + Optional<InstanceName> instance = Optional.of(deployment.applicationId().instance()); + Optional<AssignedCertificate> currentCertificate = curator.readAssignedCertificate(application, instance); + final AssignedCertificate assignedCertificate; + if (currentCertificate.isEmpty()) { + Optional<String> generatedId = Optional.empty(); + // Re-use the generated ID contained in an existing certificate (matching this application, this instance, + // or any other instance present in deployment sec), if any. If this exists we provision a new certificate + // containing the same ID + if (!deployment.zoneId().environment().isManuallyDeployed()) { + generatedId = curator.readAssignedCertificates().stream() + .filter(ac -> { + boolean matchingInstance = ac.instance().isPresent() && + deploymentSpec.instance(ac.instance().get()).isPresent(); + return (matchingInstance || ac.instance().isEmpty()) && + ac.application().equals(application); + }) + .map(AssignedCertificate::certificate) + .flatMap(ac -> ac.generatedId().stream()) + .findFirst(); + } + if (allowPool && generatedId.isEmpty()) { + assignedCertificate = assignFromPool(application, instance, deployment.zoneId()); + } else { + if (generatedId.isEmpty()) { + generatedId = Optional.of(generateId()); + } + EndpointCertificate provisionedCertificate = provision(deployment, Optional.empty(), deploymentSpec, generatedId.get()); + // We do not validate the certificate if one has never existed before - because we do not want to + // wait for it to be available before we deploy. This allows the config server to start + // provisioning nodes ASAP, and the risk is small for a new deployment. + assignedCertificate = new AssignedCertificate(application, instance, provisionedCertificate, false); + } + } else { + assignedCertificate = currentCertificate.get().withShouldValidate(!allowPool); + } + return assignedCertificate; } - private Optional<EndpointCertificate> getOrProvision(Instance instance, ZoneId zone, DeploymentSpec deploymentSpec) { - if (controller.routing().generatedEndpointsEnabled(instance.id())) { - return Optional.of(assignFromPool(instance, zone)); + private AssignedCertificate applicationLevelCertificate(DeploymentId deployment) { + if (deployment.zoneId().environment().isManuallyDeployed()) { + throw new IllegalArgumentException(deployment + " is manually deployed and cannot assign an application-level certificate"); } - Optional<AssignedCertificate> assignedCertificate = curator.readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.of(instance.id().instance())); - DeploymentId deployment = new DeploymentId(instance.id(), zone); - - if (assignedCertificate.isEmpty()) { - var provisionedCertificate = provisionEndpointCertificate(deployment, Optional.empty(), deploymentSpec); - // We do not verify the certificate if one has never existed before - because we do not want to - // wait for it to be available before we deploy. This allows the config server to start - // provisioning nodes ASAP, and the risk is small for a new deployment. - curator.writeAssignedCertificate(new AssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.of(instance.id().instance()), provisionedCertificate)); - return Optional.of(provisionedCertificate); - } else { - AssignedCertificate updated = assignedCertificate.get().with(assignedCertificate.get().certificate().withLastRequested(clock.instant().getEpochSecond())); - curator.writeAssignedCertificate(updated); + TenantAndApplicationId application = TenantAndApplicationId.from(deployment.applicationId()); + Optional<AssignedCertificate> applicationLevelCertificate = curator.readAssignedCertificate(application, Optional.empty()); + if (applicationLevelCertificate.isEmpty()) { + Optional<AssignedCertificate> instanceLevelCertificate = curator.readAssignedCertificate(application, Optional.of(deployment.applicationId().instance())); + // Migrate from instance-level certificate + if (instanceLevelCertificate.isPresent()) { + try (var transaction = new NestedTransaction()) { + AssignedCertificate assignedCertificate = instanceLevelCertificate.get().withoutInstance(); + curator.removeAssignedCertificate(application, Optional.of(deployment.applicationId().instance()), transaction); + curator.writeAssignedCertificate(assignedCertificate, transaction); + transaction.commit(); + return assignedCertificate; + } + } else { + return assignFromPool(application, Optional.empty(), deployment.zoneId()); + } + } + return applicationLevelCertificate.get(); + } + + /** Assign a certificate to given deployment. A new certificate is provisioned (possibly from a pool) and reconfigured as necessary */ + private EndpointCertificate assignTo(DeploymentId deployment, DeploymentSpec deploymentSpec, EndpointConfig config) { + // Assign certificate based on endpoint config + AssignedCertificate assignedCertificate = switch (config) { + case legacy, combined -> instanceLevelCertificate(deployment, deploymentSpec, false); + case generated -> deployment.zoneId().environment().isManuallyDeployed() + ? instanceLevelCertificate(deployment, deploymentSpec, true) + : applicationLevelCertificate(deployment); + }; + + // Generate ID if not already present in certificate + Optional<String> generatedId = assignedCertificate.certificate().generatedId(); + if (generatedId.isEmpty()) { + generatedId = Optional.of(generateId()); + assignedCertificate = assignedCertificate.with(assignedCertificate.certificate().withGeneratedId(generatedId.get())); + } + + // Ensure all wanted names are present in certificate + List<String> wantedNames = controller.routing().certificateDnsNames(deployment, deploymentSpec, generatedId.get(), config.supportsLegacy()); + Set<String> currentNames = Set.copyOf(assignedCertificate.certificate().requestedDnsSans()); + // TODO(mpolden): Consider requiring exact match for generated as we likely want to remove any legacy names in this case + if (!currentNames.containsAll(wantedNames)) { + EndpointCertificate updatedCertificate = provision(deployment, Optional.of(assignedCertificate.certificate()), deploymentSpec, generatedId.get()); + // Validation is unlikely to succeed in this case, as certificate must be available first. Controller will retry + assignedCertificate = assignedCertificate.with(updatedCertificate) + .withShouldValidate(true); } - // Re-provision certificate if it is missing SANs for the zone we are deploying to - // Skip this validation for now if the cert has a randomized id and should not provision legacy names - Optional<EndpointCertificate> currentCertificate = assignedCertificate.map(AssignedCertificate::certificate); - boolean legacyNames = assignLegacyNames.with(FetchVector.Dimension.INSTANCE_ID, instance.id().serializedForm()) - .with(FetchVector.Dimension.APPLICATION_ID, instance.id().toSerializedFormWithoutInstance()).value(); - - var requiredSansForZone = legacyNames || currentCertificate.get().generatedId().isEmpty() ? - controller.routing().certificateDnsNames(deployment, deploymentSpec) : - List.<String>of(); - - if (!currentCertificate.get().requestedDnsSans().containsAll(requiredSansForZone)) { - var reprovisionedCertificate = - provisionEndpointCertificate(deployment, currentCertificate, deploymentSpec) - .withRootRequestId(currentCertificate.get().rootRequestId()); // We're required to keep the original request ID - curator.writeAssignedCertificate(assignedCertificate.get().with(reprovisionedCertificate)); - // Verification is unlikely to succeed in this case, as certificate must be available first - controller will retry - certificateValidator.validate(reprovisionedCertificate, instance.id().serializedForm(), zone, requiredSansForZone); - return Optional.of(reprovisionedCertificate); + // Require that generated ID is always set, for any kind of certificate + if (assignedCertificate.certificate().generatedId().isEmpty()) { + throw new IllegalArgumentException("Certificate for " + deployment + " does not contain generated ID: " + + assignedCertificate.certificate()); } - certificateValidator.validate(currentCertificate.get(), instance.id().serializedForm(), zone, requiredSansForZone); - return currentCertificate; + // Update the time we last requested this certificate. This field is used by EndpointCertificateMaintainer to + // determine stale certificates + assignedCertificate = assignedCertificate.with(assignedCertificate.certificate().withLastRequested(clock.instant().getEpochSecond())); + curator.writeAssignedCertificate(assignedCertificate); + + // Validate if we're re-assigned an existing certificate, or if we updated the names of an existing certificate + if (assignedCertificate.shouldValidate()) { + certificateValidator.validate(assignedCertificate.certificate(), deployment.applicationId().serializedForm(), + deployment.zoneId(), wantedNames); + } + + return assignedCertificate.certificate(); } - private EndpointCertificate provisionEndpointCertificate(DeploymentId deployment, - Optional<EndpointCertificate> currentCert, - DeploymentSpec deploymentSpec) { + private String generateId() { + List<String> unassignedIds = curator.readUnassignedCertificates().stream() + .map(UnassignedCertificate::id) + .toList(); + List<String> assignedIds = curator.readAssignedCertificates().stream() + .map(AssignedCertificate::certificate) + .map(EndpointCertificate::generatedId) + .flatMap(Optional::stream) + .toList(); + Set<String> allIds = Stream.concat(unassignedIds.stream(), assignedIds.stream()).collect(Collectors.toSet()); + String id; + do { + id = GeneratedEndpoint.createPart(controller.random(true)); + } while (allIds.contains(id)); + return id; + } + + private EndpointCertificate provision(DeploymentId deployment, + Optional<EndpointCertificate> current, + DeploymentSpec deploymentSpec, + String generatedId) { List<ZoneId> zonesInSystem = controller.zoneRegistry().zones().controllerUpgraded().ids(); Set<ZoneId> requiredZones = new LinkedHashSet<>(); requiredZones.add(deployment.zoneId()); @@ -214,39 +283,36 @@ public class EndpointCertificates { instanceSpec.get().deploysTo(zone.environment(), zone.region()))) .forEach(requiredZones::add); } - /* TODO(andreer/mpolden): To allow a seamless transition of existing deployments to using generated endpoints, - we need to something like this: - 1) All current certificates must be re-provisioned to contain the same wildcard names - as CertificatePoolMaintainer, and a randomized ID - 2) Generated endpoints must be exposed *before* switching deployment to a - pre-provisioned certificate - 3) Tenants must shift their traffic to generated endpoints - 4) We can switch to the pre-provisioned certificate. This will invalidate - non-generated endpoints - */ - Set<String> requiredNames = requiredZones.stream() + Set<String> wantedNames = requiredZones.stream() .flatMap(zone -> controller.routing().certificateDnsNames(new DeploymentId(deployment.applicationId(), zone), - deploymentSpec) + deploymentSpec, generatedId, true) .stream()) .collect(Collectors.toCollection(LinkedHashSet::new)); - // Preserve any currently present names that are still valid - List<String> currentNames = currentCert.map(EndpointCertificate::requestedDnsSans) - .orElseGet(List::of); - zonesInSystem.stream() - .map(zone -> controller.routing().certificateDnsNames(new DeploymentId(deployment.applicationId(), zone), deploymentSpec)) - .filter(currentNames::containsAll) - .forEach(requiredNames::addAll); + // Preserve any currently present names that are still valid (i.e. the name points to a zone found in this system) + Set<String> currentNames = current.map(EndpointCertificate::requestedDnsSans) + .map(Set::copyOf) + .orElseGet(Set::of); + for (var zone : zonesInSystem) { + List<String> wantedNamesZone = controller.routing().certificateDnsNames(new DeploymentId(deployment.applicationId(), zone), + deploymentSpec, + generatedId, + true); + if (currentNames.containsAll(wantedNamesZone)) { + wantedNames.addAll(wantedNamesZone); + } + } - log.log(Level.INFO, String.format("Requesting new endpoint certificate from Cameo for application %s", deployment.applicationId().serializedForm())); - String algo = this.endpointCertificateAlgo.with(FetchVector.Dimension.INSTANCE_ID, deployment.applicationId().serializedForm()).value(); + // Request certificate + LOG.log(Level.INFO, String.format("Requesting new endpoint certificate for application %s", deployment.applicationId().serializedForm())); + String algo = endpointCertificateAlgo.with(FetchVector.Dimension.INSTANCE_ID, deployment.applicationId().serializedForm()).value(); boolean useAlternativeProvider = useAlternateCertProvider.with(FetchVector.Dimension.INSTANCE_ID, deployment.applicationId().serializedForm()).value(); String keyPrefix = deployment.applicationId().toFullString(); - var t0 = Instant.now(); - EndpointCertificate endpointCertificate = certificateProvider.requestCaSignedCertificate(keyPrefix, List.copyOf(requiredNames), currentCert, algo, useAlternativeProvider); - var t1 = Instant.now(); - log.log(Level.INFO, String.format("Endpoint certificate request for application %s returned after %s", deployment.applicationId().serializedForm(), Duration.between(t0, t1))); - return endpointCertificate; + Instant t0 = controller.clock().instant(); + EndpointCertificate endpointCertificate = certificateProvider.requestCaSignedCertificate(keyPrefix, List.copyOf(wantedNames), current, algo, useAlternativeProvider); + Instant t1 = controller.clock().instant(); + LOG.log(Level.INFO, String.format("Endpoint certificate request for application %s returned after %s", deployment.applicationId().serializedForm(), Duration.between(t0, t1))); + return endpointCertificate.withGeneratedId(generatedId); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 1b4fda28095..1080b379c4d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -273,7 +273,8 @@ public class InternalStepRunner implements StepRunner { case CERTIFICATE_NOT_READY -> { logger.log("No valid CA signed certificate for app available to config server"); if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) { - logger.log(WARNING, "CA signed certificate for app not available to config server within " + timeouts.endpointCertificate()); + logger.log(WARNING, "CA signed certificate for app not available to config server within " + + timeouts.endpointCertificate().toMinutes() + " minutes"); return Optional.of(RunStatus.endpointCertificateTimeout); } return result; @@ -330,10 +331,10 @@ public class InternalStepRunner implements StepRunner { case CERT_NOT_AVAILABLE: // Same as CERTIFICATE_NOT_READY above, only from the controller logger.log("Retrieving CA signed certificate for the application. " + - "This may take up to " + timeouts.endpointCertificate() + " on first deployment."); + "This may take up to " + timeouts.endpointCertificate().toMinutes() + " minutes on first deployment."); if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) { logger.log(WARNING, "CA signed certificate for app not available within " + - timeouts.endpointCertificate() + ": " + Exceptions.toMessageString(e)); + timeouts.endpointCertificate().toMinutes() + " minutes: " + Exceptions.toMessageString(e)); return Optional.of(RunStatus.endpointCertificateTimeout); } return Optional.empty(); @@ -726,6 +727,8 @@ public class InternalStepRunner implements StepRunner { DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); boolean requireTests = spec.steps().stream().anyMatch(step -> step.concerns(id.type().environment())); + logger.log(WARNING, "No tests were actually run, but this test suite is explicitly declared in 'deployment.xml'. " + + "Either add tests, ensure they're correctly configured, or remove the test declaration."); return Optional.of(requireTests ? testFailure : noTests); } case SUCCESS: diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java index db24eb06b48..5e6e495e473 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java @@ -2,6 +2,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import ai.vespa.metrics.ControllerMetrics; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.Metric; @@ -11,9 +14,9 @@ import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; -import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; @@ -30,7 +33,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Manages pool of ready-to-use randomized endpoint certificates + * Manages a pool of ready-to-use endpoint certificates. * * @author andreer */ @@ -44,7 +47,6 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { private final Metric metric; private final Controller controller; private final IntFlag certPoolSize; - private final String dnsSuffix; private final StringFlag endpointCertificateAlgo; private final BooleanFlag useAlternateCertProvider; @@ -58,7 +60,6 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { this.curator = controller.curator(); this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); this.metric = metric; - this.dnsSuffix = Endpoint.dnsSuffix(controller.system()); } protected double maintain() { @@ -72,10 +73,10 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? ((double)available/poolSize) : 1.0), metric.createContext(Map.of())); if (certificatePool.size() < poolSize) { - provisionRandomizedCertificate(); + provisionCertificate(); } } catch (Exception e) { - log.log(Level.SEVERE, "Exception caught while maintaining pool of unused randomized endpoint certs", e); + log.log(Level.SEVERE, "Failed to maintain certificate pool", e); return 1.0; } return 0.0; @@ -90,17 +91,17 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { OptionalInt maxCertVersion = secretStore.listSecretVersions(cert.certificate().certName()).stream().mapToInt(i -> i).max(); if (maxKeyVersion.isPresent() && maxCertVersion.equals(maxKeyVersion)) { curator.writeUnassignedCertificate(cert.withState(UnassignedCertificate.State.ready)); - log.log(Level.INFO, "Randomized endpoint cert %s now ready for use".formatted(cert.id())); + log.log(Level.INFO, "Readied certificate %s".formatted(cert.id())); } } catch (SecretNotFoundException s) { // Likely because the certificate is very recently provisioned - ignore till next time - should we log? - log.log(Level.INFO, "Could not yet read secrets for randomized endpoint cert %s - maybe next time ...".formatted(cert.id())); + log.log(Level.INFO, "Cannot ready certificate %s yet, will retry in %s".formatted(cert.id(), interval())); } } } } - private void provisionRandomizedCertificate() { + private void provisionCertificate() { try (Mutex lock = controller.curator().lockCertificatePool()) { Set<String> existingNames = controller.curator().readUnassignedCertificates().stream().map(UnassignedCertificate::id).collect(Collectors.toSet()); @@ -109,27 +110,30 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { .map(EndpointCertificate::generatedId) .forEach(id -> id.ifPresent(existingNames::add)); - String id = generateRandomId(); - while (existingNames.contains(id)) id = generateRandomId(); - - EndpointCertificate f = endpointCertificateProvider.requestCaSignedCertificate( - "preprovisioned.%s".formatted(id), - List.of( - "*.%s.z%s".formatted(id, dnsSuffix), - "*.%s.g%s".formatted(id, dnsSuffix), - "*.%s.a%s".formatted(id, dnsSuffix) - ), - Optional.empty(), - endpointCertificateAlgo.value(), - useAlternateCertProvider.value()) - .withGeneratedId(id); - - UnassignedCertificate certificate = new UnassignedCertificate(f, UnassignedCertificate.State.requested); + String id = generateId(); + while (existingNames.contains(id)) id = generateId(); + List<String> dnsNames = wildcardDnsNames(id); + EndpointCertificate cert = endpointCertificateProvider.requestCaSignedCertificate( + "preprovisioned.%s".formatted(id), + dnsNames, + Optional.empty(), + endpointCertificateAlgo.value(), + useAlternateCertProvider.value()).withGeneratedId(id); + + UnassignedCertificate certificate = new UnassignedCertificate(cert, UnassignedCertificate.State.requested); curator.writeUnassignedCertificate(certificate); } } - private String generateRandomId() { + private List<String> wildcardDnsNames(String id) { + DeploymentId defaultDeployment = new DeploymentId(ApplicationId.defaultId(), ZoneId.defaultId()); + return controller.routing().certificateDnsNames(defaultDeployment, // Not used for non-legacy names + DeploymentSpec.empty, // Not used for non-legacy names + id, + false); + } + + private String generateId() { return GeneratedEndpoint.createPart(controller.random(true)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 1d4b6e53109..e3e3e347c04 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -3,32 +3,24 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Sets; import com.yahoo.component.annotation.Inject; -import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.IntFlag; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateDetails; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateRequest; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; -import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.secrets.EndpointSecretManager; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; +import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -42,11 +34,9 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.OptionalInt; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates. @@ -66,9 +56,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { private final EndpointSecretManager endpointSecretManager; private final EndpointCertificateProvider endpointCertificateProvider; final Comparator<EligibleJob> oldestFirst = Comparator.comparing(e -> e.deployment.at()); - private final StringFlag endpointCertificateAlgo; - private final BooleanFlag useAlternateCertProvider; - private final IntFlag assignRandomizedIdRate; @Inject public EndpointCertificateMaintainer(Controller controller, Duration interval) { @@ -79,9 +66,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { this.endpointSecretManager = controller.serviceRegistry().secretManager(); this.curator = controller().curator(); this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); - this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); - this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); - this.assignRandomizedIdRate = Flags.ASSIGNED_RANDOMIZED_ID_RATE.bindTo(controller.flagSource()); } @Override @@ -92,12 +76,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { updateRefreshedCertificates(); deleteUnusedCertificates(); deleteOrReportUnmanagedCertificates(); - assignRandomizedIds(); } catch (Exception e) { log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e); return 1.0; } - return 0.0; } @@ -269,115 +251,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { } } - private void assignRandomizedIds() { - List<AssignedCertificate> assignedCertificates = curator.readAssignedCertificates(); - /* - only assign randomized id if: - * instance is present - * randomized id is not already assigned - * feature flag is enabled - */ - assignedCertificates.stream() - .filter(c -> c.instance().isPresent()) - .filter(c -> c.certificate().generatedId().isEmpty()) - .filter(c -> controller().applications().getApplication(c.application()).isPresent()) // In case application has been deleted, but certificate is pending deletion - .limit(assignRandomizedIdRate.value()) - .forEach(c -> assignRandomizedId(c.application(), c.instance().get())); - } - - /* - Assign randomized id according to these rules: - * Instance is not mentioned in the deployment spec for this application - -> assume this is a manual deployment. Assign a randomized id to the certificate, save using instance only - * Instance is mentioned in deployment spec: - -> If there is a random endpoint assigned to tenant:application -> use this also for the "instance" certificate - -> Otherwise assign a random endpoint and write to the application and the instance. - */ - private void assignRandomizedId(TenantAndApplicationId tenantAndApplicationId, InstanceName instanceName) { - Optional<AssignedCertificate> assignedCertificate = curator.readAssignedCertificate(tenantAndApplicationId, Optional.of(instanceName)); - if (assignedCertificate.isEmpty()) { - log.log(Level.INFO, "Assigned certificate missing for " + tenantAndApplicationId.instance(instanceName).toFullString() + " when assigning randomized id"); - } - // Verify that the assigned certificate still does not have randomized id assigned - if (assignedCertificate.get().certificate().generatedId().isPresent()) return; - - controller().applications().lockApplicationOrThrow(tenantAndApplicationId, application -> { - DeploymentSpec deploymentSpec = application.get().deploymentSpec(); - if (deploymentSpec.instance(instanceName).isPresent()) { - Optional<AssignedCertificate> applicationLevelAssignedCertificate = curator.readAssignedCertificate(tenantAndApplicationId, Optional.empty()); - assignApplicationRandomId(assignedCertificate.get(), applicationLevelAssignedCertificate); - } else { - assignInstanceRandomId(assignedCertificate.get()); - } - }); - } - - private void assignApplicationRandomId(AssignedCertificate instanceLevelAssignedCertificate, Optional<AssignedCertificate> applicationLevelAssignedCertificate) { - TenantAndApplicationId tenantAndApplicationId = instanceLevelAssignedCertificate.application(); - if (applicationLevelAssignedCertificate.isPresent()) { - // Application level assigned certificate with randomized id already exists. Copy randomized id to instance level certificate and request with random names. - EndpointCertificate withRandomNames = requestRandomNames( - tenantAndApplicationId, - instanceLevelAssignedCertificate.instance(), - applicationLevelAssignedCertificate.get().certificate().generatedId() - .orElseThrow(() -> new IllegalArgumentException("Application certificate already assigned to " + tenantAndApplicationId.toString() + ", but random id is missing")), - Optional.of(instanceLevelAssignedCertificate.certificate())); - AssignedCertificate assignedCertWithRandomNames = instanceLevelAssignedCertificate.with(withRandomNames); - curator.writeAssignedCertificate(assignedCertWithRandomNames); - } else { - // No application level certificate exists, generate new assigned certificate with the randomized id based names only, then request same names also for instance level cert - String randomId = generateRandomId(); - EndpointCertificate applicationLevelEndpointCert = requestRandomNames(tenantAndApplicationId, Optional.empty(), randomId, Optional.empty()); - AssignedCertificate applicationLevelCert = new AssignedCertificate(tenantAndApplicationId, Optional.empty(), applicationLevelEndpointCert); - - EndpointCertificate instanceLevelEndpointCert = requestRandomNames(tenantAndApplicationId, instanceLevelAssignedCertificate.instance(), randomId, Optional.of(instanceLevelAssignedCertificate.certificate())); - instanceLevelAssignedCertificate = instanceLevelAssignedCertificate.with(instanceLevelEndpointCert); - - // Save both in transaction - try (NestedTransaction transaction = new NestedTransaction()) { - curator.writeAssignedCertificate(instanceLevelAssignedCertificate, transaction); - curator.writeAssignedCertificate(applicationLevelCert, transaction); - transaction.commit(); - } - } - } - - private void assignInstanceRandomId(AssignedCertificate assignedCertificate) { - String randomId = generateRandomId(); - EndpointCertificate withRandomNames = requestRandomNames(assignedCertificate.application(), assignedCertificate.instance(), randomId, Optional.of(assignedCertificate.certificate())); - AssignedCertificate assignedCertWithRandomNames = assignedCertificate.with(withRandomNames); - curator.writeAssignedCertificate(assignedCertWithRandomNames); - } - - private EndpointCertificate requestRandomNames(TenantAndApplicationId tenantAndApplicationId, Optional<InstanceName> instanceName, String randomId, Optional<EndpointCertificate> previousRequest) { - String dnsSuffix = Endpoint.dnsSuffix(controller().system()); - List<String> newSanDnsEntries = List.of( - "*.%s.z%s".formatted(randomId, dnsSuffix), - "*.%s.g%s".formatted(randomId, dnsSuffix), - "*.%s.a%s".formatted(randomId, dnsSuffix)); - List<String> existingSanDnsEntries = previousRequest.map(EndpointCertificate::requestedDnsSans).orElse(List.of()); - List<String> requestNames = Stream.concat(existingSanDnsEntries.stream(), newSanDnsEntries.stream()).toList(); - String key = instanceName.map(tenantAndApplicationId::instance).map(ApplicationId::toFullString).orElseGet(tenantAndApplicationId::toString); - return endpointCertificateProvider.requestCaSignedCertificate( - key, - requestNames, - previousRequest, - endpointCertificateAlgo.value(), - useAlternateCertProvider.value()) - .withGeneratedId(randomId); - } - - private String generateRandomId() { - List<String> unassignedIds = curator.readUnassignedCertificates().stream().map(UnassignedCertificate::id).toList(); - List<String> assignedIds = curator.readAssignedCertificates().stream().map(AssignedCertificate::certificate).map(EndpointCertificate::generatedId).filter(Optional::isPresent).map(Optional::get).toList(); - Set<String> allIds = Stream.concat(unassignedIds.stream(), assignedIds.stream()).collect(Collectors.toSet()); - String randomId; - do { - randomId = GeneratedEndpoint.createPart(controller().random(true)); - } while (allIds.contains(randomId)); - return randomId; - } - private static String asString(TenantAndApplicationId application, Optional<InstanceName> instanceName) { return application.toString() + instanceName.map(name -> "." + name.value()).orElse(""); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 6f00ff39637..0f482b1a015 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -1,7 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import ai.vespa.metrics.ControllerMetrics; import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner; @@ -11,11 +13,14 @@ import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.StepRunner; import java.time.Duration; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; @@ -32,22 +37,29 @@ public class JobRunner extends ControllerMaintainer { private final JobController jobs; private final ExecutorService executors; private final StepRunner runner; + private final Metrics metrics; public JobRunner(Controller controller, Duration duration) { - this(controller, duration, Executors.newFixedThreadPool(32, new DaemonThreadFactory("job-runner-")), new InternalStepRunner(controller)); + this(controller, duration, Executors.newFixedThreadPool(32, new DaemonThreadFactory("job-runner-")), + new InternalStepRunner(controller)); } public JobRunner(Controller controller, Duration duration, ExecutorService executors, StepRunner runner) { + this(controller, duration, executors, runner, new Metrics(controller.metric(), Duration.ofMillis(100))); + } + + JobRunner(Controller controller, Duration duration, ExecutorService executors, StepRunner runner, Metrics metrics) { super(controller, duration); this.jobs = controller.jobController(); this.jobs.setRunner(this::advance); this.executors = executors; this.runner = runner; + this.metrics = metrics; } @Override protected double maintain() { - executors.execute(() -> jobs.active().forEach(this::advance)); + execute(() -> jobs.active().forEach(this::advance)); jobs.collectGarbage(); return 1.0; } @@ -55,6 +67,7 @@ public class JobRunner extends ControllerMaintainer { @Override public void shutdown() { super.shutdown(); + metrics.shutdown(); executors.shutdown(); } @@ -83,14 +96,14 @@ public class JobRunner extends ControllerMaintainer { jobs.locked(id, run -> { if ( ! run.hasFailed() && controller().clock().instant().isAfter(run.sleepUntil().orElse(run.start()).plus(jobTimeout))) - executors.execute(() -> { + execute(() -> { jobs.abort(run.id(), "job timeout of " + jobTimeout + " reached", false); advance(run.id()); }); else if (run.readySteps().isEmpty()) - executors.execute(() -> finish(run.id())); + execute(() -> finish(run.id())); else if (run.hasFailed() || run.sleepUntil().map(sleepUntil -> ! sleepUntil.isAfter(controller().clock().instant())).orElse(true)) - run.readySteps().forEach(step -> executors.execute(() -> advance(run.id(), step))); + run.readySteps().forEach(step -> execute(() -> advance(run.id(), step))); return null; }); @@ -145,4 +158,39 @@ public class JobRunner extends ControllerMaintainer { } } + private void execute(Runnable task) { + metrics.queued.incrementAndGet(); + executors.execute(() -> { + metrics.queued.decrementAndGet(); + metrics.active.incrementAndGet(); + try { task.run(); } + finally { metrics.active.decrementAndGet(); } + }); + } + + static class Metrics { + + private final AtomicInteger queued = new AtomicInteger(); + private final AtomicInteger active = new AtomicInteger(); + private final ScheduledExecutorService reporter = Executors.newSingleThreadScheduledExecutor(new DaemonThreadFactory("job-runner-metrics-")); + private final Metric metric; + private final Metric.Context context; + + Metrics(Metric metric, Duration interval) { + this.metric = metric; + this.context = metric.createContext(Map.of()); + reporter.scheduleAtFixedRate(this::report, interval.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS); + } + + void report() { + metric.set(ControllerMetrics.DEPLOYMENT_JOBS_QUEUED.baseName(), queued.get(), context); + metric.set(ControllerMetrics.DEPLOYMENT_JOBS_ACTIVE.baseName(), active.get(), context); + } + + void shutdown() { + reporter.shutdown(); + } + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index f34cb170cb3..5cadd13309b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import ai.vespa.metrics.ControllerMetrics; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; @@ -18,11 +19,9 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -199,14 +198,9 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { .filter(this::unlessNodeOwnerIsSystemApplication) .filter(this::isNodeStateMeterable) .filter(this::isClusterTypeMeterable) - // Grouping by ApplicationId -> Architecture -> ResourceSnapshot - .collect(Collectors.groupingBy(node -> - node.owner().get(), - groupSnapshotsByArchitectureAndMajorVersion(zoneId))) + .collect(groupSnapshots(zoneId)) .values() .stream() - .flatMap(byArch -> byArch.values().stream()) - .flatMap(byMajor -> byMajor.values().stream()) .toList(); } @@ -281,17 +275,15 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { )); } - private Collector<Node, ?, Map<NodeResources.Architecture, Map<Integer, ResourceSnapshot>>> groupSnapshotsByArchitectureAndMajorVersion(ZoneId zoneId) { - return Collectors.groupingBy( - (Node node) -> node.resources().architecture(), - Collectors.collectingAndThen( - Collectors.groupingBy( - (Node node) -> node.wantedVersion().getMajor(), - Collectors.toList()), - convertNodeListToResourceSnapshot(zoneId))); + private Collector<Node, ?, Map<ResourceKey, ResourceSnapshot>> groupSnapshots(ZoneId zoneId) { + return Collectors.collectingAndThen( + Collectors.groupingBy( + (Node node) -> new ResourceKey(node.owner().get(), node.resources().architecture(), node.wantedVersion().getMajor(), node.cloudAccount()), + Collectors.toList()), + convertNodeListToResourceSnapshot(zoneId)); } - private Function<Map<Integer, List<Node>>, Map<Integer, ResourceSnapshot>> convertNodeListToResourceSnapshot(ZoneId zoneId) { + private Function<Map<ResourceKey, List<Node>>, Map<ResourceKey, ResourceSnapshot>> convertNodeListToResourceSnapshot(ZoneId zoneId) { return nodesByMajor -> { return nodesByMajor.entrySet().stream() .collect(Collectors.toMap( @@ -299,4 +291,10 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { entry -> ResourceSnapshot.from(entry.getValue(), clock.instant(), zoneId))); }; } + + private record ResourceKey( + ApplicationId applicationId, + NodeResources.Architecture architecture, + int majorVersion, + CloudAccount account) {} } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index ea425d59a9f..dceb3921061 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.function.UnaryOperator; import java.util.logging.Level; @@ -42,10 +43,16 @@ public class Upgrader extends ControllerMaintainer { private static final Logger log = Logger.getLogger(Upgrader.class.getName()); private final CuratorDb curator; + private final Random random; public Upgrader(Controller controller, Duration interval) { + this(controller, interval, controller.random(false)); + } + + Upgrader(Controller controller, Duration interval, Random random) { super(controller, interval); this.curator = controller.curator(); + this.random = random; } /** @@ -75,7 +82,7 @@ public class Upgrader extends ControllerMaintainer { private InstanceList instances(DeploymentStatusList deploymentStatuses) { return InstanceList.from(deploymentStatuses) .withDeclaredJobs() - .shuffle(controller().random(false)) + .shuffle(random) .byIncreasingDeployedVersion() .unpinned(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java index d22efdc5f6e..48e9d1f6786 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java @@ -2,8 +2,11 @@ package com.yahoo.vespa.hosted.controller.notification; import java.time.Instant; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * Represents an event that we want to notify the tenant about. The message(s) should be short @@ -13,15 +16,21 @@ import java.util.Objects; * * @author freva */ -public record Notification(Instant at, com.yahoo.vespa.hosted.controller.notification.Notification.Type type, com.yahoo.vespa.hosted.controller.notification.Notification.Level level, NotificationSource source, List<String> messages) { +public record Notification(Instant at, Notification.Type type, Notification.Level level, NotificationSource source, + List<String> messages, Optional<MailContent> mailContent) { public Notification(Instant at, Type type, Level level, NotificationSource source, List<String> messages) { - this.at = Objects.requireNonNull(at, "at cannot be null"); - this.type = Objects.requireNonNull(type, "type cannot be null"); - this.level = Objects.requireNonNull(level, "level cannot be null"); - this.source = Objects.requireNonNull(source, "source cannot be null"); - this.messages = List.copyOf(Objects.requireNonNull(messages, "messages cannot be null")); + this(at, type, level, source, messages, Optional.empty()); + } + + public Notification { + at = Objects.requireNonNull(at, "at cannot be null"); + type = Objects.requireNonNull(type, "type cannot be null"); + level = Objects.requireNonNull(level, "level cannot be null"); + source = Objects.requireNonNull(source, "source cannot be null"); + messages = List.copyOf(Objects.requireNonNull(messages, "messages cannot be null")); if (messages.size() < 1) throw new IllegalArgumentException("messages cannot be empty"); + mailContent = Objects.requireNonNull(mailContent); } public enum Level { @@ -63,4 +72,36 @@ public record Notification(Instant at, com.yahoo.vespa.hosted.controller.notific } + public static class MailContent { + private final String template; + private final Map<String, Object> values; + private final String subject; + + private MailContent(Builder b) { + template = Objects.requireNonNull(b.template); + values = Map.copyOf(b.values); + subject = b.subject; + } + + public String template() { return template; } + public Map<String, Object> values() { return Map.copyOf(values); } + public Optional<String> subject() { return Optional.ofNullable(subject); } + + public static Builder fromTemplate(String template) { return new Builder(template); } + + public static class Builder { + private final String template; + private final HashMap<String, Object> values = new HashMap<>(); + private String subject; + + private Builder(String template) { + this.template = template; + } + + public Builder with(String name, Object value) { values.put(name, value); return this; } + public Builder subject(String s) { this.subject = s; return this; } + public MailContent build() { return new MailContent(this); } + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java index afb260bf765..c1e1f075552 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java @@ -16,8 +16,17 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; - +import com.yahoo.yolean.Exceptions; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.Velocity; +import org.apache.velocity.app.VelocityEngine; +import org.apache.velocity.runtime.resource.loader.StringResourceLoader; +import org.apache.velocity.runtime.resource.util.StringResourceRepository; +import org.apache.velocity.tools.generic.EscapeTool; + +import java.io.StringWriter; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.List; import java.util.Objects; @@ -26,8 +35,6 @@ import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static com.yahoo.yolean.Exceptions.uncheck; - /** * Notifier is responsible for dispatching user notifications to their chosen Contact points. * @@ -39,6 +46,7 @@ public class Notifier { private final FlagSource flagSource; private final NotificationFormatter formatter; private final URI dashboardUri; + private final VelocityEngine velocity; private static final Logger log = Logger.getLogger(Notifier.class.getName()); @@ -51,6 +59,28 @@ public class Notifier { this.flagSource = Objects.requireNonNull(flagSource); this.formatter = new NotificationFormatter(zoneRegistry); this.dashboardUri = zoneRegistry.dashboardUrl(); + this.velocity = createTemplateEngine(); + } + + private static VelocityEngine createTemplateEngine() { + var v = new VelocityEngine(); + v.setProperty(Velocity.RESOURCE_LOADERS, "string"); + v.setProperty(Velocity.RESOURCE_LOADER + ".string.class", StringResourceLoader.class.getName()); + v.setProperty(Velocity.RESOURCE_LOADER + ".string.repository.static", "false"); + v.init(); + var repo = (StringResourceRepository) v.getApplicationAttribute(StringResourceLoader.REPOSITORY_NAME_DEFAULT); + registerTemplate(repo, "mail"); + registerTemplate(repo, "default-mail-content"); + registerTemplate(repo, "notification-message"); + return v; + } + + private static void registerTemplate(StringResourceRepository repo, String name) { + var templateStr = Exceptions.uncheck(() -> { + var in = Notifier.class.getResourceAsStream("/mail/%s.vm".formatted(name)); + return new String(in.readAllBytes()); + }); + repo.putStringResource(name, templateStr); } public void dispatch(List<Notification> notifications, NotificationSource source) { @@ -114,23 +144,43 @@ public class Notifier { public Mail mailOf(FormattedNotification content, Collection<String> recipients) { var notification = content.notification(); - var subject = Text.format("[%s] %s Vespa Notification for %s", notification.level().toString().toUpperCase(), content.prettyType(), applicationIdSource(notification.source())); - var template = uncheck(() -> Notifier.class.getResourceAsStream("/mail/mail-notification.tmpl").readAllBytes()); - var html = new String(template) - .replace("[[NOTIFICATION_HEADER]]", content.messagePrefix()) - .replace("[[NOTIFICATION_ITEMS]]", notification.messages().stream() - .map(Notifier::linkify) - .map(Notifier::capitalise) - .map(m -> "<p>" + m + "</p>") - .collect(Collectors.joining())) - .replace("[[LINK_TO_NOTIFICATION]]", notificationLink(notification.source())) - .replace("[[LINK_TO_ACCOUNT_NOTIFICATIONS]]", accountNotificationsUri(content.notification().source().tenant())) - .replace("[[LINK_TO_PRIVACY_POLICY]]", "https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html") - .replace("[[LINK_TO_TERMS_OF_SERVICE]]", consoleUri("terms-of-service-trial.html")) - .replace("[[LINK_TO_SUPPORT]]", consoleUri("support")); + var subject = content.notification().mailContent().flatMap(Notification.MailContent::subject) + .orElseGet(() -> Text.format( + "[%s] %s Vespa Notification for %s", notification.level().toString().toUpperCase(), + content.prettyType(), applicationIdSource(notification.source()))); + var html = generateHtml(content); return new Mail(recipients, subject, "", html); } + private String generateHtml(FormattedNotification content) { + var esc = new EscapeTool(); + var mailContent = content.notification().mailContent().orElseGet(() -> generateContentFromMessages(content, esc)); + var ctx = new VelocityContext(); + ctx.put("esc", esc); + ctx.put("accountNotificationLink", accountNotificationsUri(content.notification().source().tenant())); + ctx.put("privacyPolicyLink", "https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html"); + ctx.put("termsOfServiceLink", consoleUri("terms-of-service-trial.html")); + ctx.put("supportLink", consoleUri("support")); + ctx.put("mailBodyTemplate", mailContent.template()); + mailContent.values().forEach(ctx::put); + + var writer = new StringWriter(); + // Ignoring return value - implementation either returns 'true' or throws, never 'false' + velocity.mergeTemplate("mail", StandardCharsets.UTF_8.name(), ctx, writer); + return writer.toString(); + } + + private Notification.MailContent generateContentFromMessages(FormattedNotification f, EscapeTool esc) { + var items = f.notification().messages().stream().map(m -> capitalise(linkify(esc.html(m)))).toList(); + return Notification.MailContent.fromTemplate("default-mail-content") + .with("mailMessageTemplate", "notification-message") + .with("mailTitle", "Vespa Cloud Notifications") + .with("notificationHeader", f.messagePrefix()) + .with("notificationItems", items) + .with("consoleLink", notificationLink(f.notification().source())) + .build(); + } + @VisibleForTesting static String linkify(String text) { return urlPattern.matcher(text).replaceAll((res) -> String.format("<a href=\"%s\">%s</a>", res.group(), res.group())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 7923dbb34e3..a2a4cf809b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -6,7 +6,6 @@ import com.yahoo.component.Version; import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.ClusterSpec.Id; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; @@ -643,6 +642,10 @@ public class CuratorDb { curator.delete(endpointCertificatePath(application, instanceName)); } + public void removeAssignedCertificate(TenantAndApplicationId application, Optional<InstanceName> instanceName, NestedTransaction transaction) { + transaction.add(CuratorTransaction.from(CuratorOperations.delete(endpointCertificatePath(application, instanceName).getAbsolute()), curator)); + } + // TODO(mpolden): Remove this. Caller should make an explicit decision to read certificate for a particular instance public Optional<AssignedCertificate> readAssignedCertificate(ApplicationId applicationId) { return readAssignedCertificate(TenantAndApplicationId.from(applicationId), Optional.of(applicationId.instance())); @@ -651,7 +654,7 @@ public class CuratorDb { public Optional<AssignedCertificate> readAssignedCertificate(TenantAndApplicationId application, Optional<InstanceName> instance) { return readSlime(endpointCertificatePath(application, instance)).map(Slime::get) .map(EndpointCertificateSerializer::fromSlime) - .map(cert -> new AssignedCertificate(application, instance, cert)); + .map(cert -> new AssignedCertificate(application, instance, cert, false)); } public List<AssignedCertificate> readAssignedCertificates() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index 03e4a5f4378..ac3a8f2ee23 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -424,6 +424,9 @@ public class BillingApiHandler extends ThreadedHttpRequestHandler { cursor.setLong("majorVersion", lineItem.getMajorVersion()); + if (! lineItem.getCloudAccount().isUnspecified()) + cursor.setString("cloudAccount", lineItem.getCloudAccount().account()); + lineItem.getCpuHours().ifPresent(cpuHours -> cursor.setString("cpuHours", cpuHours.toString()) ); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index b49c69adad4..da83073609d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -351,6 +351,8 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler toSlime(slime.setObject("plan"), planRegistry.plan(item.plan()).orElseThrow(() -> new RuntimeException("No such plan: '" + item.plan() + "'"))); item.getArchitecture().ifPresent(arch -> slime.setString("architecture", arch.name())); slime.setLong("majorVersion", item.getMajorVersion()); + if (! item.getCloudAccount().isUnspecified()) + slime.setString("cloudAccount", item.getCloudAccount().account()); item.applicationId().ifPresent(appId -> { slime.setString("application", appId.application().value()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java index b264f3ea7c5..b38bb73a98a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.EndpointCertificateSerializer; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import java.util.List; import java.util.Optional; @@ -74,11 +75,11 @@ public class EndpointCertificatesHandler extends ThreadedHttpRequestHandler { public StringResponse reRequestEndpointCertificateFor(String instanceId, boolean ignoreExisting) { ApplicationId applicationId = ApplicationId.fromFullString(instanceId); - if (controller.routing().generatedEndpointsEnabled(applicationId)) { + if (controller.routing().endpointConfig(applicationId) == EndpointConfig.generated) { throw new IllegalArgumentException("Cannot re-request certificate. " + instanceId + " is assigned certificate from a pool"); } try (var lock = curator.lock(TenantAndApplicationId.from(applicationId))) { - AssignedCertificate assignedCertificate = curator.readAssignedCertificate(applicationId) + AssignedCertificate assignedCertificate = curator.readAssignedCertificate(TenantAndApplicationId.from(applicationId), Optional.of(applicationId.instance())) .orElseThrow(() -> new RestApiException.NotFound("No certificate found for application " + applicationId.serializedForm())); String algo = this.endpointCertificateAlgo.with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java index b6b3c8584fd..2d22ef86dce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.pricing; import com.yahoo.collections.Pair; @@ -16,25 +16,27 @@ import com.yahoo.slime.Slime; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; +import com.yahoo.vespa.hosted.controller.api.integration.pricing.ApplicationResources; import com.yahoo.vespa.hosted.controller.api.integration.pricing.PriceInformation; import com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.yolean.Exceptions; import java.math.BigDecimal; +import java.math.RoundingMode; import java.net.URLDecoder; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.logging.Logger; -import java.util.stream.Collectors; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; import static com.yahoo.restapi.ErrorResponse.methodNotAllowed; import static com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo.SupportLevel; import static java.lang.Double.parseDouble; import static java.lang.Integer.parseInt; +import static java.math.BigDecimal.valueOf; import static java.nio.charset.StandardCharsets.UTF_8; /** @@ -46,7 +48,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; public class PricingApiHandler extends ThreadedHttpRequestHandler { private static final Logger log = Logger.getLogger(PricingApiHandler.class.getName()); - private static final BigDecimal SCALED_ZERO = BigDecimal.ZERO.setScale(2); private final Controller controller; @@ -80,14 +81,30 @@ public class PricingApiHandler extends ThreadedHttpRequestHandler { private HttpResponse pricing(HttpRequest request) { String rawQuery = request.getUri().getRawQuery(); - PriceInformation price = parseQuery(rawQuery); - return response(price); + var priceParameters = parseQuery(rawQuery); + PriceInformation price = calculatePrice(priceParameters); + return response(price, priceParameters); } - private PriceInformation parseQuery(String rawQuery) { - String[] elements = URLDecoder.decode(rawQuery, UTF_8).split("&"); - if (elements.length == 0) throw new IllegalArgumentException("no price information found in query"); + private PriceInformation calculatePrice(PriceParameters priceParameters) { + var priceCalculator = controller.serviceRegistry().pricingController(); + if (priceParameters.appResources == null) + return priceCalculator.price(priceParameters.clusterResources, priceParameters.pricingInfo, priceParameters.plan); + else + return priceCalculator.priceForApplications(priceParameters.appResources, priceParameters.pricingInfo, priceParameters.plan); + } + + private PriceParameters parseQuery(String rawQuery) { + if (rawQuery == null) throw new IllegalArgumentException("No price information found in query"); + List<String> elements = Arrays.stream(URLDecoder.decode(rawQuery, UTF_8).split("&")).toList(); + if (keysAndValues(elements).stream().map(Pair::getFirst).toList().contains("resources")) + return parseQueryLegacy(elements); + else + return parseQuery(elements); + } + + private PriceParameters parseQueryLegacy(List<String> elements) { var supportLevel = SupportLevel.BASIC; var enclave = false; var committedSpend = 0d; @@ -95,25 +112,49 @@ public class PricingApiHandler extends ThreadedHttpRequestHandler { List<ClusterResources> clusterResources = new ArrayList<>(); for (Pair<String, String> entry : keysAndValues(elements)) { - switch (entry.getFirst()) { - case "committedSpend" -> committedSpend = parseDouble(entry.getSecond()); + switch (entry.getFirst().toLowerCase()) { + case "committedspend" -> committedSpend = parseDouble(entry.getSecond()); case "enclave" -> enclave = Boolean.parseBoolean(entry.getSecond()); - case "planId" -> plan = plan(entry.getSecond()) + case "planid" -> plan = plan(entry.getSecond()) .orElseThrow(() -> new IllegalArgumentException("Unknown plan id " + entry.getSecond())); - case "supportLevel" -> supportLevel = SupportLevel.valueOf(entry.getSecond().toUpperCase()); + case "supportlevel" -> supportLevel = SupportLevel.valueOf(entry.getSecond().toUpperCase()); case "resources" -> clusterResources.add(clusterResources(entry.getSecond())); + default -> throw new IllegalArgumentException("Unknown query parameter '" + entry.getFirst() + '\''); } } - if (clusterResources.size() < 1) throw new IllegalArgumentException("No cluster resources found in query"); + if (clusterResources.isEmpty()) throw new IllegalArgumentException("No cluster resources found in query"); PricingInfo pricingInfo = new PricingInfo(enclave, supportLevel, committedSpend); - return controller.serviceRegistry().pricingController().price(clusterResources, pricingInfo, plan); + return new PriceParameters(clusterResources, pricingInfo, plan, null); + } + + private PriceParameters parseQuery(List<String> elements) { + var supportLevel = SupportLevel.BASIC; + var enclave = false; + var committedSpend = 0d; + var applicationName = "default"; + var plan = controller.serviceRegistry().planRegistry().defaultPlan(); // fallback to default plan if not supplied + List<ApplicationResources> appResources = new ArrayList<>(); + + for (Pair<String, String> entry : keysAndValues(elements)) { + switch (entry.getFirst().toLowerCase()) { + case "committedspend" -> committedSpend = parseDouble(entry.getSecond()); + case "planid" -> plan = plan(entry.getSecond()) + .orElseThrow(() -> new IllegalArgumentException("Unknown plan id " + entry.getSecond())); + case "supportlevel" -> supportLevel = SupportLevel.valueOf(entry.getSecond().toUpperCase()); + case "application" -> appResources.add(applicationResources(entry.getSecond())); + default -> throw new IllegalArgumentException("Unknown query parameter '" + entry.getFirst() + '\''); + } + } + if (appResources.isEmpty()) throw new IllegalArgumentException("No application resources found in query"); + + // TODO: enclave does not make sense in PricingInfo anymore, remove when legacy method is removed + PricingInfo pricingInfo = new PricingInfo(false, supportLevel, committedSpend); + return new PriceParameters(List.of(), pricingInfo, plan, appResources); } private ClusterResources clusterResources(String resourcesString) { - String[] elements = resourcesString.split(","); - if (elements.length == 0) - throw new IllegalArgumentException("nothing found in cluster resources: " + resourcesString); + List<String> elements = Arrays.stream(resourcesString.split(",")).toList(); var nodes = 0; var vcpu = 0d; @@ -122,12 +163,13 @@ public class PricingApiHandler extends ThreadedHttpRequestHandler { var gpuMemoryGb = 0d; for (var element : keysAndValues(elements)) { - switch (element.getFirst()) { + switch (element.getFirst().toLowerCase()) { case "nodes" -> nodes = parseInt(element.getSecond()); case "vcpu" -> vcpu = parseDouble(element.getSecond()); - case "memoryGb" -> memoryGb = parseDouble(element.getSecond()); - case "diskGb" -> diskGb = parseDouble(element.getSecond()); - case "gpuMemoryGb" -> gpuMemoryGb = parseDouble(element.getSecond()); + case "memorygb" -> memoryGb = parseDouble(element.getSecond()); + case "diskgb" -> diskGb = parseDouble(element.getSecond()); + case "gpumemorygb" -> gpuMemoryGb = parseDouble(element.getSecond()); + default -> throw new IllegalArgumentException("Unknown resource type '" + element.getFirst() + '\''); } } @@ -137,40 +179,91 @@ public class PricingApiHandler extends ThreadedHttpRequestHandler { return new ClusterResources(nodes, 1, nodeResources); } - private List<Pair<String, String>> keysAndValues(String[] elements) { - return Arrays.stream(elements).map(element -> { + private ApplicationResources applicationResources(String appResourcesString) { + List<String> elements = Arrays.stream(appResourcesString.split(",")).toList(); + + var applicationName = "default"; + var vcpu = 0d; + var memoryGb = 0d; + var diskGb = 0d; + var gpuMemoryGb = 0d; + var enclaveVcpu = 0d; + var enclaveMemoryGb = 0d; + var enclaveDiskGb = 0d; + var enclaveGpuMemoryGb = 0d; + + for (var element : keysAndValues(elements)) { + switch (element.getFirst().toLowerCase()) { + case "name" -> applicationName = element.getSecond(); + + case "vcpu" -> vcpu = parseDouble(element.getSecond()); + case "memorygb" -> memoryGb = parseDouble(element.getSecond()); + case "diskgb" -> diskGb = parseDouble(element.getSecond()); + case "gpumemorygb" -> gpuMemoryGb = parseDouble(element.getSecond()); + + case "enclavevcpu" -> enclaveVcpu = parseDouble(element.getSecond()); + case "enclavememorygb" -> enclaveMemoryGb = parseDouble(element.getSecond()); + case "enclavediskgb" -> enclaveDiskGb = parseDouble(element.getSecond()); + case "enclavegpumemorygb" -> enclaveGpuMemoryGb = parseDouble(element.getSecond()); + + default -> throw new IllegalArgumentException("Unknown key '" + element.getFirst() + '\''); + } + } + + return new ApplicationResources(applicationName, + valueOf(vcpu), valueOf(memoryGb), valueOf(diskGb), valueOf(gpuMemoryGb), + valueOf(enclaveVcpu), valueOf(enclaveMemoryGb), valueOf(enclaveDiskGb), valueOf(enclaveGpuMemoryGb)); + } + + private List<Pair<String, String>> keysAndValues(List<String> elements) { + return elements.stream().map(element -> { var index = element.indexOf("="); - if (index <= 0 ) throw new IllegalArgumentException("Error in query parameter, expected '=' between key and value: " + element); + if (index <= 0 || index == element.length() - 1) + throw new IllegalArgumentException("Error in query parameter, expected '=' between key and value: '" + element + '\''); return new Pair<>(element.substring(0, index), element.substring(index + 1)); }) - .collect(Collectors.toList()); + .toList(); } private Optional<Plan> plan(String element) { return controller.serviceRegistry().planRegistry().plan(element); } - private static SlimeJsonResponse response(PriceInformation priceInfo) { + private static SlimeJsonResponse response(PriceInformation priceInfo, PriceParameters priceParameters) { var slime = new Slime(); Cursor cursor = slime.setObject(); var array = cursor.setArray("priceInfo"); - addItem(array, "List price", priceInfo.listPrice()); + addItem(array, supportLevelDescription(priceParameters), priceInfo.listPriceWithSupport()); addItem(array, "Enclave discount", priceInfo.enclaveDiscount()); addItem(array, "Volume discount", priceInfo.volumeDiscount()); addItem(array, "Committed spend", priceInfo.committedAmountDiscount()); - cursor.setString("totalAmount", priceInfo.totalAmount().toPlainString()); + setBigDecimal(cursor, "totalAmount", priceInfo.totalAmount()); return new SlimeJsonResponse(slime); } + private static String supportLevelDescription(PriceParameters priceParameters) { + String supportLevel = priceParameters.pricingInfo.supportLevel().name(); + return supportLevel.substring(0,1).toUpperCase() + supportLevel.substring(1).toLowerCase() + " support unit price"; + } + private static void addItem(Cursor array, String name, BigDecimal amount) { if (amount.compareTo(BigDecimal.ZERO) != 0) { var o = array.addObject(); o.setString("description", name); - o.setString("amount", SCALED_ZERO.add(amount).toPlainString()); + setBigDecimal(o, "amount", amount); } } + private static void setBigDecimal(Cursor cursor, String name, BigDecimal value) { + cursor.setString(name, value.setScale(2, RoundingMode.HALF_UP).toPlainString()); + } + + private record PriceParameters(List<ClusterResources> clusterResources, PricingInfo pricingInfo, Plan plan, + List<ApplicationResources> appResources) { + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java new file mode 100644 index 00000000000..555fd024e47 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +/** + * Endpoint configurations supported for an application. + * + * @author mpolden + */ +public enum EndpointConfig { + + /** Only legacy endpoints will be published in DNS. Certificate will contain both legacy and generated names, and is never assigned from a pool */ + legacy, + + /** Legacy and generated endpoints will be published in DNS. Certificate will contain both legacy and generated names, and is never assigned from a pool */ + combined, + + /** Only generated endpoints will be published in DNS. Certificate will contain generated names only. Certificate is assigned from a pool */ + generated; + + /** Returns whether this config supports legacy endpoints */ + public boolean supportsLegacy() { + return this == legacy || this == combined; + } + + /** Returns whether this config supports generated endpoints */ + public boolean supportsGenerated() { + return this == combined || this == generated; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java index 78a2be3bc5b..63b17a087f2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java @@ -28,13 +28,13 @@ import java.util.stream.Collectors; public record PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List<AssignedRotation> rotations, - Optional<EndpointCertificate> certificate) { + EndpointCertificate certificate) { - public PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List<AssignedRotation> rotations, Optional<EndpointCertificate> certificate) { + public PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List<AssignedRotation> rotations, EndpointCertificate certificate) { this.deployment = Objects.requireNonNull(deployment); this.endpoints = Objects.requireNonNull(endpoints); this.rotations = List.copyOf(Objects.requireNonNull(rotations)); - this.certificate = Objects.requireNonNull(certificate); + this.certificate = requireMatchingSans(certificate, endpoints); } /** Returns the endpoints contained in this as {@link com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint} */ @@ -101,4 +101,15 @@ public record PreparedEndpoints(DeploymentId deployment, }; } + private static EndpointCertificate requireMatchingSans(EndpointCertificate certificate, EndpointList endpoints) { + Objects.requireNonNull(certificate); + for (var endpoint : endpoints.not().scope(Endpoint.Scope.weighted)) { // Weighted endpoints are not present in certificate + if (!certificate.sanMatches(endpoint.dnsName())) { + throw new IllegalArgumentException(endpoint + " has no matching SAN. Certificate contains " + + certificate.requestedDnsSans()); + } + } + return certificate; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java index 2cad499899c..50e65187835 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java @@ -45,7 +45,7 @@ public abstract class DeploymentRoutingContext implements RoutingContext { * * @return the container endpoints relevant for this deployment, as declared in deployment spec */ - public final PreparedEndpoints prepare(BasicServicesXml services, Optional<EndpointCertificate> certificate, LockedApplication application) { + public final PreparedEndpoints prepare(BasicServicesXml services, EndpointCertificate certificate, LockedApplication application) { return routing.prepare(deployment, services, certificate, application); } diff --git a/controller-server/src/main/resources/mail/default-mail-content.vm b/controller-server/src/main/resources/mail/default-mail-content.vm new file mode 100644 index 00000000000..02de98b900d --- /dev/null +++ b/controller-server/src/main/resources/mail/default-mail-content.vm @@ -0,0 +1,131 @@ +<tbody> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + <h1 + style=" + text-align: center; + color: #000000; + line-height: 32px; + " + > + $esc.html($mailTitle) + </h1> + </div> + </td> +</tr> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + + #parse($mailMessageTemplate) + + </div> + </td> +</tr> +<tr> + <td + align="center" + vertical-align="middle" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 20px; + padding-bottom: 20px; + word-break: break-word; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="border-collapse: separate; line-height: 100%" + > + <tbody> + <tr> + <td + align="center" + bgcolor="#005A8E" + role="presentation" + style=" + border: none; + border-radius: 100px; + cursor: auto; + mso-padding-alt: 15px 25px 15px 25px; + background: #005a8e; + " + valign="middle" + > + <a + href="$consoleLink" + style=" + display: inline-block; + background: #005a8e; + color: #ffffff; + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + font-weight: normal; + line-height: 120%; + margin: 0; + text-decoration: none; + text-transform: none; + padding: 15px 25px 15px 25px; + mso-padding-alt: 0px; + border-radius: 100px; + " + target="_blank" + ><b style="font-weight: 700" + ><b style="font-weight: 700" + >Go to Console</b + ></b + ></a + > + </td> + </tr> + </tbody> + </table> + </td> +</tr> +</tbody>
\ No newline at end of file diff --git a/controller-server/src/main/resources/mail/mail-notification.tmpl b/controller-server/src/main/resources/mail/mail.vm index 5bf5530b433..1dbec781b3a 100644 --- a/controller-server/src/main/resources/mail/mail-notification.tmpl +++ b/controller-server/src/main/resources/mail/mail.vm @@ -383,138 +383,9 @@ style="vertical-align: top" width="100%" > - <tbody> - <tr> - <td - align="left" - style=" - font-size: 0px; - padding: 0px 25px 0px 25px; - padding-top: 0px; - padding-right: 50px; - padding-bottom: 0px; - padding-left: 50px; - word-break: break-word; - " - > - <div - style=" - font-family: Open Sans, Helvetica, Arial, - sans-serif; - font-size: 13px; - line-height: 22px; - text-align: left; - color: #797e82; - " - > - <h1 - style=" - text-align: center; - color: #000000; - line-height: 32px; - " - > - Vespa Cloud Notifications - </h1> - </div> - </td> - </tr> - <tr> - <td - align="left" - style=" - font-size: 0px; - padding: 0px 25px 0px 25px; - padding-top: 0px; - padding-right: 50px; - padding-bottom: 0px; - padding-left: 50px; - word-break: break-word; - " - > - <div - style=" - font-family: Open Sans, Helvetica, Arial, - sans-serif; - font-size: 13px; - line-height: 22px; - text-align: left; - color: #797e82; - " - > - <p> - [[NOTIFICATION_HEADER]]: - </p> - [[NOTIFICATION_ITEMS]] - </div> - </td> - </tr> - <tr> - <td - align="center" - vertical-align="middle" - style=" - font-size: 0px; - padding: 10px 25px; - padding-top: 20px; - padding-bottom: 20px; - word-break: break-word; - " - > - <table - border="0" - cellpadding="0" - cellspacing="0" - role="presentation" - style="border-collapse: separate; line-height: 100%" - > - <tbody> - <tr> - <td - align="center" - bgcolor="#005A8E" - role="presentation" - style=" - border: none; - border-radius: 100px; - cursor: auto; - mso-padding-alt: 15px 25px 15px 25px; - background: #005a8e; - " - valign="middle" - > - <a - href="[[LINK_TO_NOTIFICATION]]" - style=" - display: inline-block; - background: #005a8e; - color: #ffffff; - font-family: Open Sans, Helvetica, Arial, - sans-serif; - font-size: 13px; - font-weight: normal; - line-height: 120%; - margin: 0; - text-decoration: none; - text-transform: none; - padding: 15px 25px 15px 25px; - mso-padding-alt: 0px; - border-radius: 100px; - " - target="_blank" - ><b style="font-weight: 700" - ><b style="font-weight: 700" - >Go to Console</b - ></b - ></a - > - </td> - </tr> - </tbody> - </table> - </td> - </tr> - </tbody> + + #parse($mailBodyTemplate) + </table> </div> <!--[if mso | IE]></td></tr></table><![endif]--> @@ -592,7 +463,7 @@ target="_blank" rel="noopener noreferrer" style="color: #005a8e" - href="[[LINK_TO_PRIVACY_POLICY]]" + href="$privacyPolicyLink" ><span style="color: #005a8e" >Yahoo Privacy Policy</span ></a @@ -602,7 +473,7 @@ target="_blank" rel="noopener noreferrer" style="color: #005a8e" - href="[[LINK_TO_TERMS_OF_SERVICE]]" + href="$termsOfServiceLink" ><span style="color: #005a8e" >Terms of Service</span ></a @@ -612,7 +483,7 @@ target="_blank" rel="noopener noreferrer" style="color: #005a8e" - href="[[LINK_TO_SUPPORT]]" + href="$supportLink" ><span style="color: #005a8e">Support</span></a > </p> @@ -621,7 +492,7 @@ target="_blank" rel="noopener noreferrer" style="color: inherit; text-decoration: none" - href="[[LINK_TO_ACCOUNT_NOTIFICATIONS]]" + href="$accountNotificationLink" >Click <span style="color: #005a8e"><u>here</u></span> to manage your notifications setting.</a diff --git a/controller-server/src/main/resources/mail/notification-message.vm b/controller-server/src/main/resources/mail/notification-message.vm new file mode 100644 index 00000000000..29673d38420 --- /dev/null +++ b/controller-server/src/main/resources/mail/notification-message.vm @@ -0,0 +1,6 @@ +<p> + $esc.html($notificationHeader): +</p> +#foreach( $i in $notificationItems ) +<p>$i</p> +#end diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index f456b6e12dc..eb86f23fbfb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -73,7 +73,6 @@ import java.util.Set; import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devAwsUsEast2a; @@ -952,8 +951,6 @@ public class ControllerTest { // Create app1 var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); var prodZone = ZoneId.from("prod", "us-west-1"); - var stagingZone = ZoneId.from("staging", "us-east-3"); - var testZone = ZoneId.from("test", "us-east-1"); tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(prodZone)); var applicationPackage = new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) .region(prodZone.region()) @@ -962,16 +959,23 @@ public class ControllerTest { context1.submit(applicationPackage).deploy(); var cert = certificate.apply(context1.instance()); assertTrue(cert.isPresent(), "Provisions certificate in " + Environment.prod); - assertEquals(Stream.concat(Stream.of("vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud", - "app1.tenant1.global.vespa.oath.cloud", - "*.app1.tenant1.global.vespa.oath.cloud"), - Stream.of(prodZone, testZone, stagingZone) - .flatMap(zone -> Stream.of("", "*.") - .map(prefix -> prefix + "app1.tenant1." + zone.region().value() + - (zone.environment() == Environment.prod ? "" : "." + zone.environment().value()) + - ".vespa.oath.cloud"))) - .collect(Collectors.toUnmodifiableSet()), - Set.copyOf(tester.controllerTester().serviceRegistry().endpointCertificateMock().dnsNamesOf(cert.get().rootRequestId()))); + assertEquals(List.of("*.app1.tenant1.global.vespa.oath.cloud", + "*.app1.tenant1.us-east-1.test.vespa.oath.cloud", + "*.app1.tenant1.us-east-3.staging.vespa.oath.cloud", + "*.app1.tenant1.us-west-1.vespa.oath.cloud", + "*.f5549014.a.vespa.oath.cloud", + "*.f5549014.g.vespa.oath.cloud", + "*.f5549014.z.vespa.oath.cloud", + "app1.tenant1.global.vespa.oath.cloud", + "app1.tenant1.us-east-1.test.vespa.oath.cloud", + "app1.tenant1.us-east-3.staging.vespa.oath.cloud", + "app1.tenant1.us-west-1.vespa.oath.cloud", + "vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud"), + tester.controllerTester().serviceRegistry().endpointCertificateMock() + .dnsNamesOf(cert.get().rootRequestId()) + .stream() + .sorted() + .toList()); // Next deployment reuses certificate context1.submit(applicationPackage).deploy(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 40162186066..7bdecca11c0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -3,9 +3,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -20,6 +18,7 @@ import com.yahoo.vespa.athenz.api.OAuthCredentials; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock; @@ -54,7 +53,6 @@ import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import com.yahoo.yolean.concurrent.Sleeper; @@ -289,14 +287,6 @@ public final class ControllerTester { return controller().clock().instant().atOffset(ZoneOffset.UTC).getHour(); } - public ZoneId toZone(Environment environment) { - return switch (environment) { - case dev, test -> ZoneId.from(environment, RegionName.from("us-east-1")); - case staging -> ZoneId.from(environment, RegionName.from("us-east-3")); - default -> ZoneId.from(environment, RegionName.from("us-west-1")); - }; - } - public AthenzDomain createDomainWithAdmin(String domainName, AthenzUser user) { AthenzDomain domain = new AthenzDomain(domainName); athenzDb.getOrCreateDomain(domain).admin(user); @@ -405,7 +395,7 @@ public final class ControllerTester { RotationsConfig.Builder builder = new RotationsConfig.Builder(); for (int i = 1; i <= availableRotations; i++) { String id = Text.format("%02d", i); - builder = builder.rotations("rotation-id-" + id, "rotation-fqdn-" + id); + builder.rotations("rotation-id-" + id, "rotation-fqdn-" + id); } return new RotationsConfig(builder); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index 23481cb324e..cec48dd1598 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -370,6 +370,24 @@ public class EndpointTest { "dead2bad.deadbeef.a.vespa-app.cloud", Endpoint.of(TenantAndApplicationId.from(instance1)).targetApplication(EndpointId.of("foo"), deployment) .generatedFrom(ge2) + .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public), + // Wildcard endpoint for zone + "*.deadbeef.z.vespa-app.cloud", + Endpoint.of(instance1) + .wildcardGenerated(ge1.applicationPart(), Endpoint.Scope.zone) + .certificateName() + .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public), + // Wildcard endpoint for global + "*.deadbeef.g.vespa-app.cloud", + Endpoint.of(instance1) + .wildcardGenerated(ge1.applicationPart(), Endpoint.Scope.global) + .certificateName() + .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public), + // Wildcard endpoint for application + "*.deadbeef.a.vespa-app.cloud", + Endpoint.of(instance1) + .wildcardGenerated(ge1.applicationPart(), Endpoint.Scope.application) + .certificateName() .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.dnsName())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index 78748cd2cd8..7faaee95abb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -17,18 +17,21 @@ import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.test.ManualClock; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProviderMock; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateValidatorImpl; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateValidatorMock; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -58,12 +61,13 @@ public class EndpointCertificatesTest { private final ControllerTester tester = new ControllerTester(); private final SecretStoreMock secretStore = new SecretStoreMock(); - private final CuratorDb mockCuratorDb = tester.curator(); + private final CuratorDb curator = tester.curator(); private final ManualClock clock = tester.clock(); private final EndpointCertificateProviderMock endpointCertificateProviderMock = new EndpointCertificateProviderMock(); private final EndpointCertificateValidatorImpl endpointCertificateValidator = new EndpointCertificateValidatorImpl(secretStore, clock); private final EndpointCertificates endpointCertificates = new EndpointCertificates(tester.controller(), endpointCertificateProviderMock, endpointCertificateValidator); private final KeyPair testKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 192); + private final Mutex lock = () -> {}; private X509Certificate testCertificate; private X509Certificate testCertificate2; @@ -74,6 +78,9 @@ public class EndpointCertificatesTest { "*.default.default.global.vespa.oath.cloud", "default.default.aws-us-east-1a.vespa.oath.cloud", "*.default.default.aws-us-east-1a.vespa.oath.cloud", + "*.f5549014.z.vespa.oath.cloud", + "*.f5549014.g.vespa.oath.cloud", + "*.f5549014.a.vespa.oath.cloud", "default.default.us-east-1.test.vespa.oath.cloud", "*.default.default.us-east-1.test.vespa.oath.cloud", "default.default.us-east-3.staging.vespa.oath.cloud", @@ -93,7 +100,10 @@ public class EndpointCertificatesTest { private static final List<String> expectedDevSans = List.of( "vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", "default.default.us-east-1.dev.vespa.oath.cloud", - "*.default.default.us-east-1.dev.vespa.oath.cloud" + "*.default.default.us-east-1.dev.vespa.oath.cloud", + "*.f5549014.z.vespa.oath.cloud", + "*.f5549014.g.vespa.oath.cloud", + "*.f5549014.a.vespa.oath.cloud" ); private X509Certificate makeTestCert(List<String> sans) { @@ -108,7 +118,7 @@ public class EndpointCertificatesTest { return x509CertificateBuilder.build(); } - private final Instance instance = new Instance(ApplicationId.defaultId()); + private final ApplicationId instance = ApplicationId.defaultId(); private final String testKeyName = "testKeyName"; private final String testCertName = "testCertName"; private ZoneId prodZone; @@ -125,22 +135,20 @@ public class EndpointCertificatesTest { @Test void provisions_new_certificate_in_dev() { ZoneId testZone = tester.zoneRegistry().zones().all().routingMethod(RoutingMethod.exclusive).in(Environment.dev).zones().stream().findFirst().orElseThrow().getId(); - Optional<EndpointCertificate> cert = endpointCertificates.get(instance, testZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertTrue(cert.get().keyName().matches("vespa.tls.default.default.*-key")); - assertTrue(cert.get().certName().matches("vespa.tls.default.default.*-cert")); - assertEquals(0, cert.get().version()); - assertEquals(expectedDevSans, cert.get().requestedDnsSans()); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, testZone), DeploymentSpec.empty, lock); + assertTrue(cert.keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(cert.certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, cert.version()); + assertEquals(expectedDevSans, cert.requestedDnsSans()); } @Test void provisions_new_certificate_in_prod() { - Optional<EndpointCertificate> cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertTrue(cert.get().keyName().matches("vespa.tls.default.default.*-key")); - assertTrue(cert.get().certName().matches("vespa.tls.default.default.*-cert")); - assertEquals(0, cert.get().version()); - assertEquals(expectedSans, cert.get().requestedDnsSans()); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); + assertTrue(cert.keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(cert.certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, cert.version()); + assertEquals(expectedSans, cert.requestedDnsSans()); } private ControllerTester publicTester() { @@ -160,66 +168,68 @@ public class EndpointCertificatesTest { "*.default.default.g.vespa-app.cloud", "default.default.aws-us-east-1a.z.vespa-app.cloud", "*.default.default.aws-us-east-1a.z.vespa-app.cloud", + "*.f5549014.z.vespa-app.cloud", + "*.f5549014.g.vespa-app.cloud", + "*.f5549014.a.vespa-app.cloud", "default.default.us-east-1.test.z.vespa-app.cloud", "*.default.default.us-east-1.test.z.vespa-app.cloud", "default.default.us-east-3.staging.z.vespa-app.cloud", "*.default.default.us-east-3.staging.z.vespa-app.cloud" ); - Optional<EndpointCertificate> cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertTrue(cert.get().keyName().matches("vespa.tls.default.default.*-key")); - assertTrue(cert.get().certName().matches("vespa.tls.default.default.*-cert")); - assertEquals(0, cert.get().version()); - assertEquals(expectedSans, cert.get().requestedDnsSans()); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); + assertTrue(cert.keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(cert.certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, cert.version()); + assertEquals(expectedSans, cert.requestedDnsSans()); } @Test void reuses_stored_certificate() { - mockCuratorDb.writeAssignedCertificate(assignedCertificate(instance.id(), new EndpointCertificate(testKeyName, testCertName, 7, 0, "request_id", Optional.of("leaf-request-uuid"), - List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", + curator.writeAssignedCertificate(assignedCertificate(instance, new EndpointCertificate(testKeyName, testCertName, 7, 0, "request_id", Optional.of("leaf-request-uuid"), + List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", "default.default.global.vespa.oath.cloud", "*.default.default.global.vespa.oath.cloud", "default.default.aws-us-east-1a.vespa.oath.cloud", - "*.default.default.aws-us-east-1a.vespa.oath.cloud"), - "", Optional.empty(), Optional.empty(), Optional.empty()))); + "*.default.default.aws-us-east-1a.vespa.oath.cloud", + "*.f5549014.z.vespa.oath.cloud", + "*.f5549014.g.vespa.oath.cloud", + "*.f5549014.a.vespa.oath.cloud"), + "", Optional.empty(), Optional.empty(), Optional.empty()))); secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 7); secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 7); - Optional<EndpointCertificate> cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertEquals(testKeyName, cert.get().keyName()); - assertEquals(testCertName, cert.get().certName()); - assertEquals(7, cert.get().version()); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); + assertEquals(testKeyName, cert.keyName()); + assertEquals(testCertName, cert.certName()); + assertEquals(7, cert.version()); } @Test void reprovisions_certificate_when_necessary() { - mockCuratorDb.writeAssignedCertificate(assignedCertificate(instance.id(), new EndpointCertificate(testKeyName, testCertName, -1, 0, "root-request-uuid", Optional.of("leaf-request-uuid"), List.of(), "issuer", Optional.empty(), Optional.empty(), Optional.empty()))); + curator.writeAssignedCertificate(assignedCertificate(instance, new EndpointCertificate(testKeyName, testCertName, -1, 0, "root-request-uuid", Optional.of("leaf-request-uuid"), List.of(), "issuer", Optional.empty(), Optional.empty(), Optional.empty()))); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 0); - Optional<EndpointCertificate> cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertEquals(0, cert.get().version()); - assertEquals(cert, mockCuratorDb.readAssignedCertificate(instance.id()).map(AssignedCertificate::certificate)); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); + assertEquals(0, cert.version()); + assertEquals(cert, curator.readAssignedCertificate(instance).map(AssignedCertificate::certificate).get()); } @Test void reprovisions_certificate_with_added_sans_when_deploying_to_new_zone() { ZoneId testZone = ZoneId.from("prod.ap-northeast-1"); - mockCuratorDb.writeAssignedCertificate(assignedCertificate(instance.id(), new EndpointCertificate(testKeyName, testCertName, -1, 0, "original-request-uuid", Optional.of("leaf-request-uuid"), expectedSans, "mockCa", Optional.empty(), Optional.empty(), Optional.empty()))); + curator.writeAssignedCertificate(assignedCertificate(instance, new EndpointCertificate(testKeyName, testCertName, -1, 0, "original-request-uuid", Optional.of("leaf-request-uuid"), expectedSans, "mockCa", Optional.empty(), Optional.empty(), Optional.empty()))); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), -1); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), -1); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate2) + X509CertificateUtils.toPem(testCertificate2), 0); - Optional<EndpointCertificate> cert = endpointCertificates.get(instance, testZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertEquals(0, cert.get().version()); - assertEquals(cert, mockCuratorDb.readAssignedCertificate(instance.id()).map(AssignedCertificate::certificate)); - assertEquals("original-request-uuid", cert.get().rootRequestId()); - assertNotEquals(Optional.of("leaf-request-uuid"), cert.get().leafRequestId()); - assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(cert.get().requestedDnsSans())); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, testZone), DeploymentSpec.empty, lock); + assertEquals(0, cert.version()); + assertEquals(cert, curator.readAssignedCertificate(instance).map(AssignedCertificate::certificate).get()); + assertEquals("original-request-uuid", cert.rootRequestId()); + assertNotEquals(Optional.of("leaf-request-uuid"), cert.leafRequestId()); + assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(cert.requestedDnsSans())); } @Test @@ -238,17 +248,16 @@ public class EndpointCertificatesTest { ); ZoneId testZone = tester.zoneRegistry().zones().all().in(Environment.staging).zones().stream().findFirst().orElseThrow().getId(); - Optional<EndpointCertificate> cert = endpointCertificates.get(instance, testZone, deploymentSpec); - assertTrue(cert.isPresent()); - assertTrue(cert.get().keyName().matches("vespa.tls.default.default.*-key")); - assertTrue(cert.get().certName().matches("vespa.tls.default.default.*-cert")); - assertEquals(0, cert.get().version()); - assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(cert.get().requestedDnsSans())); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, testZone), deploymentSpec, lock); + assertTrue(cert.keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(cert.certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, cert.version()); + assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(cert.requestedDnsSans())); } @Test void includes_application_endpoint_when_declared() { - Instance instance = new Instance(ApplicationId.from("t1", "a1", "default")); + ApplicationId instance = ApplicationId.from("t1", "a1", "default"); ZoneId zone1 = ZoneId.from(Environment.prod, RegionName.from("aws-us-east-1c")); ZoneId zone2 = ZoneId.from(Environment.prod, RegionName.from("aws-us-west-2a")); ControllerTester tester = publicTester(); @@ -280,28 +289,25 @@ public class EndpointCertificatesTest { "a1.t1.us-east-1.test.z.vespa-app.cloud", "*.a1.t1.us-east-1.test.z.vespa-app.cloud", "a1.t1.us-east-3.staging.z.vespa-app.cloud", - "*.a1.t1.us-east-3.staging.z.vespa-app.cloud" + "*.a1.t1.us-east-3.staging.z.vespa-app.cloud", + "*.f5549014.z.vespa-app.cloud", + "*.f5549014.g.vespa-app.cloud", + "*.f5549014.a.vespa-app.cloud" ).sorted().toList(); - Optional<EndpointCertificate> cert = endpointCertificates.get(instance, zone1, applicationPackage.deploymentSpec()); - assertTrue(cert.isPresent()); - assertTrue(cert.get().keyName().matches("vespa.tls.t1.a1.*-key")); - assertTrue(cert.get().certName().matches("vespa.tls.t1.a1.*-cert")); - assertEquals(0, cert.get().version()); - assertEquals(expectedSans, cert.get().requestedDnsSans().stream().sorted().toList()); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, zone1), applicationPackage.deploymentSpec(), lock); + assertTrue(cert.keyName().matches("vespa.tls.t1.a1.*-key")); + assertTrue(cert.certName().matches("vespa.tls.t1.a1.*-cert")); + assertEquals(0, cert.version()); + assertEquals(expectedSans, cert.requestedDnsSans().stream().sorted().toList()); } @Test public void assign_certificate_from_pool() { - // Initial certificate is requested directly from provider - Optional<EndpointCertificate> certFromProvider = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(certFromProvider.isPresent()); - assertFalse(certFromProvider.get().generatedId().isPresent()); - - // Pooled certificates become available tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); try { - addCertificateToPool("pool-cert-1", UnassignedCertificate.State.requested); - endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); + addCertificateToPool("bad0f00d", UnassignedCertificate.State.requested, tester); + endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); fail("Expected exception as certificate is not ready"); } catch (IllegalArgumentException ignored) {} @@ -311,76 +317,169 @@ public class EndpointCertificatesTest { // Certificate is assigned from pool instead. The previously assigned certificate will eventually be cleaned up // by EndpointCertificateMaintainer { // prod - String certId = "pool-cert-1"; - addCertificateToPool(certId, UnassignedCertificate.State.ready); - Optional<EndpointCertificate> cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertEquals(certId, cert.get().generatedId().get()); - assertEquals(certId, tester.curator().readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.empty()).get().certificate().generatedId().get(), "Certificate is assigned at application-level"); + String certId = "bad0f00d"; + addCertificateToPool(certId, UnassignedCertificate.State.ready, tester); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); + assertEquals(certId, cert.generatedId().get()); + assertEquals(certId, tester.curator().readAssignedCertificate(TenantAndApplicationId.from(instance), Optional.empty()).get().certificate().generatedId().get(), "Certificate is assigned at application-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); - assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); + assertEquals(clock.instant().getEpochSecond(), cert.lastRequested()); } { // dev - String certId = "pool-cert-2"; - addCertificateToPool(certId, UnassignedCertificate.State.ready); + String certId = "f00d0bad"; + addCertificateToPool(certId, UnassignedCertificate.State.ready, tester); ZoneId devZone = tester.zoneRegistry().zones().all().routingMethod(RoutingMethod.exclusive).in(Environment.dev).zones().stream().findFirst().orElseThrow().getId(); - Optional<EndpointCertificate> cert = endpointCertificates.get(instance, devZone, DeploymentSpec.empty); - assertEquals(certId, cert.get().generatedId().get()); - assertEquals(certId, tester.curator().readAssignedCertificate(instance.id()).get().certificate().generatedId().get(), "Certificate is assigned at instance-level"); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, devZone), DeploymentSpec.empty, lock); + assertEquals(certId, cert.generatedId().get()); + assertEquals(certId, tester.curator().readAssignedCertificate(instance).get().certificate().generatedId().get(), "Certificate is assigned at instance-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); - assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); + assertEquals(clock.instant().getEpochSecond(), cert.lastRequested()); } } @Test - void reuse_per_instance_certificate_if_assigned_random_id() { - // Initial certificate is requested directly from provider - Optional<EndpointCertificate> certFromProvider = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(certFromProvider.isPresent()); - assertFalse(certFromProvider.get().generatedId().isPresent()); - - // Simulate endpoint certificate maintainer to assign random id - TenantAndApplicationId tenantAndApplicationId = TenantAndApplicationId.from(instance.id()); - Optional<InstanceName> instanceName = Optional.of(instance.name()); - Optional<AssignedCertificate> assignedCertificate = tester.controller().curator().readAssignedCertificate(tenantAndApplicationId, instanceName); - assertTrue(assignedCertificate.isPresent()); - String assignedRandomId = "randomid"; - AssignedCertificate updated = assignedCertificate.get().with(assignedCertificate.get().certificate().withGeneratedId(assignedRandomId)); - tester.controller().curator().writeAssignedCertificate(updated); - - // Pooled certificates become available + public void certificate_migration() { + // An application is initially deployed with legacy config + ZoneId zone1 = ZoneId.from(Environment.prod, RegionName.from("aws-us-east-1c")); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder().region(zone1.region()) + .build(); + ControllerTester tester = publicTester(); + EndpointCertificates endpointCertificates = new EndpointCertificates(tester.controller(), endpointCertificateProviderMock, new EndpointCertificateValidatorMock()); + ApplicationId instance = ApplicationId.from("t1", "a1", "default"); + DeploymentId deployment0 = new DeploymentId(instance, zone1); + final EndpointCertificate certificate = endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock); + final String generatedId = certificate.generatedId().get(); + assertEquals(List.of("vlfms2wpoa4nyrka2s5lktucypjtxkqhv.internal.vespa-app.cloud", + "a1.t1.g.vespa-app.cloud", + "*.a1.t1.g.vespa-app.cloud", + "a1.t1.aws-us-east-1c.z.vespa-app.cloud", + "*.a1.t1.aws-us-east-1c.z.vespa-app.cloud", + "*.f5549014.z.vespa-app.cloud", + "*.f5549014.g.vespa-app.cloud", + "*.f5549014.a.vespa-app.cloud", + "a1.t1.us-east-1.test.z.vespa-app.cloud", + "*.a1.t1.us-east-1.test.z.vespa-app.cloud", + "a1.t1.us-east-3.staging.z.vespa-app.cloud", + "*.a1.t1.us-east-3.staging.z.vespa-app.cloud"), + certificate.requestedDnsSans()); + Optional<AssignedCertificate> assignedCertificate = tester.curator().readAssignedCertificate(deployment0.applicationId()); + assertTrue(assignedCertificate.isPresent(), "Certificate is assigned at instance level"); + assertTrue(assignedCertificate.get().certificate().generatedId().isPresent(), "Certificate contains generated ID"); + + // Re-requesting certificate does not make any changes, except last requested time + tester.clock().advance(Duration.ofHours(1)); + assertEquals(certificate.withLastRequested(tester.clock().instant().getEpochSecond()), + endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock), + "Next request returns same certificate and updates last requested time"); + + // An additional instance is added to deployment spec + applicationPackage = new ApplicationPackageBuilder().instances("default,beta") + .region(zone1.region()) + .build(); + DeploymentId deployment1 = new DeploymentId(ApplicationId.from("t1", "a1", "beta"), zone1); + EndpointCertificate betaCert = endpointCertificates.get(deployment1, applicationPackage.deploymentSpec(), lock); + assertEquals(List.of("v43ctkgqim52zsbwefrg6ixkuwidvsumy.internal.vespa-app.cloud", + "beta.a1.t1.g.vespa-app.cloud", + "*.beta.a1.t1.g.vespa-app.cloud", + "beta.a1.t1.aws-us-east-1c.z.vespa-app.cloud", + "*.beta.a1.t1.aws-us-east-1c.z.vespa-app.cloud", + "*.f5549014.z.vespa-app.cloud", + "*.f5549014.g.vespa-app.cloud", + "*.f5549014.a.vespa-app.cloud", + "beta.a1.t1.us-east-1.test.z.vespa-app.cloud", + "*.beta.a1.t1.us-east-1.test.z.vespa-app.cloud", + "beta.a1.t1.us-east-3.staging.z.vespa-app.cloud", + "*.beta.a1.t1.us-east-3.staging.z.vespa-app.cloud"), + betaCert.requestedDnsSans()); + assertEquals(generatedId, betaCert.generatedId().get(), "Certificate inherits generated ID of existing instance"); + + // A dev instance is deployed + DeploymentId devDeployment0 = new DeploymentId(ApplicationId.from("t1", "a1", "dev"), + ZoneId.from("dev", "us-east-1")); + EndpointCertificate devCert0 = endpointCertificates.get(devDeployment0, applicationPackage.deploymentSpec(), lock); + assertNotEquals(generatedId, devCert0.generatedId().get(), "Dev deployments gets a new generated ID"); + assertEquals(List.of("vld3y4mggzpd5wmm5jmldzcbyetjoqtzq.internal.vespa-app.cloud", + "dev.a1.t1.us-east-1.dev.z.vespa-app.cloud", + "*.dev.a1.t1.us-east-1.dev.z.vespa-app.cloud", + "*.a89ff7c6.z.vespa-app.cloud", + "*.a89ff7c6.g.vespa-app.cloud", + "*.a89ff7c6.a.vespa-app.cloud"), + devCert0.requestedDnsSans()); + + // Application switches to combined config tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); - - // Create 1 cert in pool - String certId = "pool-cert-1"; - addCertificateToPool(certId, UnassignedCertificate.State.ready); - - // Request cert for app - Optional<EndpointCertificate> cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertEquals(assignedRandomId, cert.get().generatedId().get()); - - // Pooled cert remains unassigned - List<String> unassignedCertificateIds = tester.curator().readUnassignedCertificates().stream() - .map(UnassignedCertificate::certificate) - .map(EndpointCertificate::generatedId) - .map(Optional::get) - .toList(); - assertEquals(List.of(certId), unassignedCertificateIds); + tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), true); + tester.clock().advance(Duration.ofHours(1)); + assertEquals(certificate.withLastRequested(tester.clock().instant().getEpochSecond()), + endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock), + "No change to certificate: Existing certificate is compatible with " + + EndpointConfig.combined + " config"); + assertTrue(tester.curator().readAssignedCertificate(deployment0.applicationId()).isPresent(), "Certificate is assigned at instance level"); + assertFalse(tester.curator().readAssignedCertificate(TenantAndApplicationId.from(deployment0.applicationId()), Optional.empty()).isPresent(), + "Certificate is not assigned at application level"); + + // Application switches to generated config + tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); + tester.clock().advance(Duration.ofHours(1)); + assertEquals(certificate.withLastRequested(tester.clock().instant().getEpochSecond()), + endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock), + "No change to certificate: Existing certificate is compatible with " + + EndpointConfig.generated + " config"); + assertFalse(tester.curator().readAssignedCertificate(deployment0.applicationId()).isPresent(), "Certificate is no longer assigned at instance level"); + assertTrue(tester.curator().readAssignedCertificate(TenantAndApplicationId.from(deployment0.applicationId()), Optional.empty()).isPresent(), + "Certificate is assigned at application level"); + + // Both instances still use the same certificate + assertEquals(endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock), + endpointCertificates.get(deployment1, applicationPackage.deploymentSpec(), lock)); + + // Another dev instance is deployed, and is assigned certificate from pool + String poolCertId0 = "badf00d0"; + addCertificateToPool(poolCertId0, UnassignedCertificate.State.ready, tester); + EndpointCertificate devCert1 = endpointCertificates.get(new DeploymentId(ApplicationId.from("t1", "a1", "dev2"), + ZoneId.from("dev", "us-east-1")), + applicationPackage.deploymentSpec(), lock); + assertEquals(poolCertId0, devCert1.generatedId().get()); + + // Another application is deployed, and is assigned certificate from pool + String poolCertId1 = "badf00d1"; + addCertificateToPool(poolCertId1, UnassignedCertificate.State.ready, tester); + EndpointCertificate prodCertificate = endpointCertificates.get(new DeploymentId(ApplicationId.from("t1", "a2", "default"), + ZoneId.from("prod", "us-east-1")), + applicationPackage.deploymentSpec(), lock); + assertEquals(poolCertId1, prodCertificate.generatedId().get()); + + // Application switches back to legacy config + tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), false); + tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), true); + EndpointCertificate reissuedCertificate = endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock); + assertEquals(certificate.requestedDnsSans(), reissuedCertificate.requestedDnsSans()); + assertTrue(tester.curator().readAssignedCertificate(deployment0.applicationId()).isPresent(), "Certificate is assigned at instance level again"); + assertTrue(tester.curator().readAssignedCertificate(TenantAndApplicationId.from(deployment0.applicationId()), Optional.empty()).isPresent(), + "Certificate is still assigned at application level"); // Not removed because the assumption is that the application will eventually migrate back } - private void addCertificateToPool(String id, UnassignedCertificate.State state) { - EndpointCertificate cert = new EndpointCertificate(testKeyName, testCertName, 1, 0, + private void addCertificateToPool(String id, UnassignedCertificate.State state, ControllerTester tester) { + EndpointCertificate cert = new EndpointCertificate(testKeyName, + testCertName, + 1, + 0, "request-id", Optional.of("leaf-request-uuid"), - List.of("name1", "name2"), - "", Optional.empty(), - Optional.empty(), Optional.of(id)); + List.of("*." + id + ".z.vespa.oath.cloud", + "*." + id + ".g.vespa.oath.cloud", + "*." + id + ".a.vespa.oath.cloud"), + "", + Optional.empty(), + Optional.empty(), + Optional.of(id)); UnassignedCertificate pooledCert = new UnassignedCertificate(cert, state); tester.controller().curator().writeUnassignedCertificate(pooledCert); } private static AssignedCertificate assignedCertificate(ApplicationId instance, EndpointCertificate certificate) { - return new AssignedCertificate(TenantAndApplicationId.from(instance), Optional.of(instance.instance()), certificate); + return new AssignedCertificate(TenantAndApplicationId.from(instance), Optional.of(instance.instance()), certificate, false); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 1cc549ec6ca..c16234b3948 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -75,8 +75,7 @@ public class DeploymentTester { tester = controllerTester; jobs = tester.controller().jobController(); cloud = (MockTesterCloud) tester.controller().jobController().cloud(); - runner = new JobRunner(tester.controller(), maintenanceInterval, JobRunnerTest.inThreadExecutor(), - new InternalStepRunner(tester.controller())); + runner = new JobRunner(tester.controller(), maintenanceInterval, JobRunnerTest.inThreadExecutor(), new InternalStepRunner(tester.controller())); upgrader = new Upgrader(tester.controller(), maintenanceInterval); upgrader.setUpgradesPerMinute(1); // Anything that makes it at least one for any maintenance period is fine. readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index bb36aa01b0f..f551a99829e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -45,7 +45,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.pro import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -120,6 +119,7 @@ public class EndpointCertificateMaintainerTest { var applicationPackage = new ApplicationPackageBuilder() .region("us-west-1") + .container("default") .build(); DeploymentContext deploymentContext = deploymentTester.newDeploymentContext("tenant", "application", "default"); @@ -191,68 +191,6 @@ public class EndpointCertificateMaintainerTest { } @Test - void production_deployment_certificates_are_assigned_random_id() { - var app = ApplicationId.from("tenant", "app", "default"); - DeploymentTester deploymentTester = new DeploymentTester(tester); - deployToAssignCert(deploymentTester, app, List.of(systemTest, stagingTest, productionUsWest1), Optional.empty()); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - - maintainer.maintain(); - assertEquals(2, tester.curator().readAssignedCertificates().size()); - - // Verify random id is same for application and instance certificates - Optional<AssignedCertificate> applicationCertificate = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(app), Optional.empty()); - assertTrue(applicationCertificate.isPresent()); - Optional<AssignedCertificate> instanceCertificate = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(app), Optional.of(app.instance())); - assertTrue(instanceCertificate.isPresent()); - assertEquals(instanceCertificate.get().certificate().generatedId(), applicationCertificate.get().certificate().generatedId()); - - // Verify the 3 wildcard random names are same in all certs - List<String> appWildcardSans = applicationCertificate.get().certificate().requestedDnsSans(); - assertEquals(3, appWildcardSans.size()); - List<String> instanceSans = instanceCertificate.get().certificate().requestedDnsSans(); - List<String> wildcards = instanceSans.stream().filter(appWildcardSans::contains).toList(); - assertEquals(appWildcardSans, wildcards); - } - - @Test - void existing_application_randomid_is_copied_to_new_instance_deployments() { - var instance1 = ApplicationId.from("tenant", "prod", "instance1"); - var instance2 = ApplicationId.from("tenant", "prod", "instance2"); - - DeploymentTester deploymentTester = new DeploymentTester(tester); - deployToAssignCert(deploymentTester, instance1, List.of(systemTest, stagingTest,productionUsWest1),Optional.of("instance1")); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - maintainer.maintain(); - - String randomId = tester.curator().readAssignedCertificate(instance1).get().certificate().generatedId().get(); - - deployToAssignCert(deploymentTester, instance2, List.of(productionUsWest1), Optional.of("instance1,instance2")); - maintainer.maintain(); - assertEquals(3, tester.curator().readAssignedCertificates().size()); - - assertEquals(randomId, tester.curator().readAssignedCertificate(instance1).get().certificate().generatedId().get()); - } - - @Test - void dev_certificates_are_not_assigned_application_level_certificate() { - var devApp = ApplicationId.from("tenant", "devonly", "foo"); - DeploymentTester deploymentTester = new DeploymentTester(tester); - deployToAssignCert(deploymentTester, devApp, List.of(devUsEast1), Optional.empty()); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - List<String> originalRequestedSans = tester.curator().readAssignedCertificate(devApp).get().certificate().requestedDnsSans(); - maintainer.maintain(); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - - // Verify certificate is assigned random id and 3 new names - Optional<AssignedCertificate> assignedCertificate = tester.curator().readAssignedCertificate(devApp); - assertTrue(assignedCertificate.get().certificate().generatedId().isPresent()); - List<String> newRequestedSans = assignedCertificate.get().certificate().requestedDnsSans(); - List<String> randomizedNames = newRequestedSans.stream().filter(san -> !originalRequestedSans.contains(san)).toList(); - assertEquals(3, randomizedNames.size()); - } - - @Test void deploy_to_other_manual_zone_refreshes_cert() { String devSan = "*.foo.manual.tenant.us-east-1.dev.vespa.oath.cloud"; String perfSan = "*.foo.manual.tenant.us-east-3.perf.vespa.oath.cloud"; @@ -301,10 +239,6 @@ public class EndpointCertificateMaintainerTest { Assertions.assertThat(usCentralWestSans).contains(centralSan); } - private void deploy() { - - } - private void deployToAssignCert(DeploymentTester tester, ApplicationId applicationId, List<JobType> jobTypes, Optional<String> instances) { var applicationPackageBuilder = new ApplicationPackageBuilder(); @@ -322,19 +256,15 @@ public class EndpointCertificateMaintainerTest { jobs.forEach(deploymentContext::runJob); } - EndpointCertificate certificate(List<String> sans) { - return new EndpointCertificate("keyName", "certName", 0, 0, "root-request-uuid", Optional.of("leaf-request-uuid"), List.of(), "issuer", Optional.empty(), Optional.empty(), Optional.empty()); - } - - private static AssignedCertificate assignedCertificate(ApplicationId instance, EndpointCertificate certificate) { - return new AssignedCertificate(TenantAndApplicationId.from(instance), Optional.of(instance.instance()), certificate); + return new AssignedCertificate(TenantAndApplicationId.from(instance), Optional.of(instance.instance()), certificate, false); } private void prepareCertificatePool(int numCertificates) { ((InMemoryFlagSource)tester.controller().flagSource()).withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), numCertificates); ((InMemoryFlagSource)tester.controller().flagSource()).withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + ((InMemoryFlagSource)tester.controller().flagSource()).withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); // Provision certificates for (int i = 0; i < numCertificates; i++) { @@ -351,4 +281,5 @@ public class EndpointCertificateMaintainerTest { }); certificatePoolMaintainer.maintain(); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index e87d4f1f3f0..20717be598f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -1,8 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import ai.vespa.metrics.ControllerMetrics; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.jdisc.test.MockMetric; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; @@ -22,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.deployment.StepRunner; import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; +import com.yahoo.vespa.hosted.controller.maintenance.JobRunner.Metrics; import org.junit.jupiter.api.Test; import java.time.Duration; @@ -37,7 +40,9 @@ import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Phaser; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -121,6 +126,51 @@ public class JobRunnerTest { } @Test + void metrics() { + Phaser phaser = new Phaser(4); + StepRunner runner = (step, id) -> { + phaser.arriveAndAwaitAdvance(); + phaser.arriveAndAwaitAdvance(); + return Optional.of(running); + }; + ExecutorService executor = new ThreadPoolExecutor(3, 3, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), (task, pool) -> task.run()); + DeploymentTester tester = new DeploymentTester(); + MockMetric metric = new MockMetric(); + Metrics metrics = new Metrics(metric, Duration.ofDays(1)); + JobRunner jobs = new JobRunner(tester.controller(), Duration.ofDays(1), executor, runner, metrics); + tester.newDeploymentContext().submit(); + + assertEquals(Map.of(), metric.metrics()); + metrics.report(); + assertEquals(Map.of(ControllerMetrics.DEPLOYMENT_JOBS_QUEUED.baseName(), + Map.of(Map.of(), 0.0), + ControllerMetrics.DEPLOYMENT_JOBS_ACTIVE.baseName(), + Map.of(Map.of(), 0.0)), + metric.metrics()); + tester.triggerJobs(); + + assertEquals(2, tester.jobs().active().size()); + jobs.maintain(); + phaser.arriveAndAwaitAdvance(); + metrics.report(); + assertEquals(Map.of(ControllerMetrics.DEPLOYMENT_JOBS_QUEUED.baseName(), + Map.of(Map.of(), 1.0), + ControllerMetrics.DEPLOYMENT_JOBS_ACTIVE.baseName(), + Map.of(Map.of(), 3.0)), + metric.metrics()); + + jobs.shutdown(); + phaser.forceTermination(); + jobs.awaitShutdown(); + metrics.report(); + assertEquals(Map.of(ControllerMetrics.DEPLOYMENT_JOBS_QUEUED.baseName(), + Map.of(Map.of(), 0.0), + ControllerMetrics.DEPLOYMENT_JOBS_ACTIVE.baseName(), + Map.of(Map.of(), 0.0)), + metric.metrics()); + } + + @Test void stepLogic() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index 3f3400b5d1b..d93dcf71317 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; @@ -62,9 +64,9 @@ public class ResourceMeterMaintainerTest { .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().cost().getAsDouble()))); List<ResourceSnapshot> resourceSnapshots = List.of( - new ResourceSnapshot(app1, resources(12, 34, 56), Instant.EPOCH, z1, 0), - new ResourceSnapshot(app1, resources(23, 45, 67), Instant.EPOCH, z2, 0), - new ResourceSnapshot(app2, resources(34, 56, 78), Instant.EPOCH, z1, 0)); + new ResourceSnapshot(app1, resources(12, 34, 56), Instant.EPOCH, z1, 0, CloudAccount.empty), + new ResourceSnapshot(app1, resources(23, 45, 67), Instant.EPOCH, z2, 0, CloudAccount.empty), + new ResourceSnapshot(app2, resources(34, 56, 78), Instant.EPOCH, z1, 0, CloudAccount.empty)); maintainer.updateDeploymentCost(resourceSnapshots); assertCost.accept(app1, Map.of(z1, 1.72, z2, 3.05)); @@ -72,9 +74,9 @@ public class ResourceMeterMaintainerTest { // Remove a region from app1 and add region to app2 resourceSnapshots = List.of( - new ResourceSnapshot(app1, resources(23, 45, 67), Instant.EPOCH, z2, 0), - new ResourceSnapshot(app2, resources(34, 56, 78), Instant.EPOCH, z1, 0), - new ResourceSnapshot(app2, resources(45, 67, 89), Instant.EPOCH, z2, 0)); + new ResourceSnapshot(app1, resources(23, 45, 67), Instant.EPOCH, z2, 0, CloudAccount.empty), + new ResourceSnapshot(app2, resources(34, 56, 78), Instant.EPOCH, z1, 0, CloudAccount.empty), + new ResourceSnapshot(app2, resources(45, 67, 89), Instant.EPOCH, z2, 0, CloudAccount.empty)); maintainer.updateDeploymentCost(resourceSnapshots); assertCost.accept(app1, Map.of(z2, 3.05)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 6882f43f1a7..c8853c008f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.Change; @@ -26,6 +25,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Random; import java.util.Set; import java.util.stream.Collectors; @@ -40,7 +40,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.Cha import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -1100,8 +1099,9 @@ public class UpgraderTest { default2.instanceId(), default2); // Throttle upgrades per run - ((ManualClock) tester.controller().clock()).setInstant(Instant.ofEpochMilli(1589787107000L)); // Fixed random seed - Upgrader upgrader = new Upgrader(tester.controller(), Duration.ofMinutes(10)); + Upgrader upgrader = new Upgrader(tester.controller(), + Duration.ofMinutes(10), + new Random(1589787107000L)); // Fixed random seed upgrader.setUpgradesPerMinute(0.1); // Trigger some upgrades diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java index 1251963f01c..f64ed3740d2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java @@ -75,7 +75,7 @@ public class NotifierTest { var mail = mailer.inbox(email.getEmailAddress()).get(0); assertEquals("[WARNING] Test package Vespa Notification for tenant1.default.default", mail.subject()); - assertEquals(new String(NotifierTest.class.getResourceAsStream("/mail/notification.txt").readAllBytes()), mail.htmlMessage().get()); + assertEquals(new String(NotifierTest.class.getResourceAsStream("/mail/notification.html").readAllBytes()), mail.htmlMessage().get()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index d0a0276b362..eb023aa9fe9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.application.container.handler.Request; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; @@ -162,7 +163,7 @@ public class ControllerApiTest extends ControllerContainerTest { new NodeResources(12, 48, 1200, 0, NodeResources.DiskSpeed.any, NodeResources.StorageType.any, NodeResources.Architecture.arm64), new NodeResources(24, 96, 2400, 0, NodeResources.DiskSpeed.any, NodeResources.StorageType.any, NodeResources.Architecture.x86_64)); - var snapshots = resources.stream().map(x -> new ResourceSnapshot(applicationId, x, timestamp, zoneId, 0)).toList(); + var snapshots = resources.stream().map(x -> new ResourceSnapshot(applicationId, x, timestamp, zoneId, 0, CloudAccount.empty)).toList(); tester.controller().serviceRegistry().resourceDatabase().writeResourceSnapshots(snapshots); tester.assertResponse( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java index 09ff6bbc4b1..521aa7cc28d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java @@ -1,13 +1,16 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.pricing; import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; import org.junit.jupiter.api.Test; import java.net.URLEncoder; +import static com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo.SupportLevel.BASIC; +import static com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo.SupportLevel.COMMERCIAL; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -19,47 +22,160 @@ public class PricingApiHandlerTest extends ControllerContainerCloudTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/responses/"; @Test - void testPricingInfo() { + void testPricingInfoBasicLegacy() { ContainerTester tester = new ContainerTester(container, responseFiles); assertEquals(SystemName.Public, tester.controller().system()); - var request = request("/pricing/v1/pricing?" + urlEncodedPriceInformation()); + var request = request("/pricing/v1/pricing?" + urlEncodedPriceInformationLegacy(BASIC, false)); tester.assertJsonResponse(request, """ { "priceInfo": [ - {"description": "List price", "amount": "2400.00"}, - {"description": "Volume discount", "amount": "-5.00"} + {"description": "Basic support unit price", "amount": "2240.00"}, + {"description": "Volume discount", "amount": "-5.64"}, + {"description": "Committed spend", "amount": "-1.23"} ], - "totalAmount": "2395.00" + "totalAmount": "2233.13" } """, 200); } @Test - void testPricingInfoWithIncompleteParameter() { + void testPricingInfoBasic() { ContainerTester tester = new ContainerTester(container, responseFiles); assertEquals(SystemName.Public, tester.controller().system()); - var request = request("/pricing/v1/pricing?" + urlEncodedPriceInformationWithMissingValueInResourcs()); - tester.assertJsonResponse(request, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Error in query parameter, expected '=' between key and value: resources\"}", - 400); + var request = request("/pricing/v1/pricing?" + urlEncodedPriceInformation1App(BASIC)); + tester.assertJsonResponse(request, """ + { + "priceInfo": [ + {"description": "Basic support unit price", "amount": "2240.00"}, + {"description": "Volume discount", "amount": "-5.64"}, + {"description": "Committed spend", "amount": "-1.23"} + ], + "totalAmount": "2233.13" + } + """, + 200); + } + + @Test + void testPricingInfoBasicEnclave() { + ContainerTester tester = new ContainerTester(container, responseFiles); + assertEquals(SystemName.Public, tester.controller().system()); + + var request = request("/pricing/v1/pricing?" + urlEncodedPriceInformation1AppEnclave(BASIC)); + tester.assertJsonResponse(request, """ + { + "priceInfo": [ + {"description": "Basic support unit price", "amount": "2240.00"}, + {"description": "Enclave discount", "amount": "-15.12"}, + {"description": "Volume discount", "amount": "-5.64"}, + {"description": "Committed spend", "amount": "-1.23"} + ], + "totalAmount": "2218.00" + } + """, + 200); + } + + @Test + void testPricingInfoCommercialEnclaveLegacy() { + ContainerTester tester = new ContainerTester(container, responseFiles); + assertEquals(SystemName.Public, tester.controller().system()); + + var request = request("/pricing/v1/pricing?" + urlEncodedPriceInformationLegacy(COMMERCIAL, true)); + tester.assertJsonResponse(request, """ + { + "priceInfo": [ + {"description": "Commercial support unit price", "amount": "3200.00"}, + {"description": "Enclave discount", "amount": "-15.12"}, + {"description": "Volume discount", "amount": "-5.64"}, + {"description": "Committed spend", "amount": "-1.23"} + ], + "totalAmount": "3178.00" + } + """, + 200); + } + + @Test + void testPricingInfoCommercialEnclave() { + ContainerTester tester = new ContainerTester(container, responseFiles); + assertEquals(SystemName.Public, tester.controller().system()); + + var request = request("/pricing/v1/pricing?" + urlEncodedPriceInformation1AppEnclave(COMMERCIAL)); + tester.assertJsonResponse(request, """ + { + "priceInfo": [ + {"description": "Commercial support unit price", "amount": "3200.00"}, + {"description": "Enclave discount", "amount": "-15.12"}, + {"description": "Volume discount", "amount": "-5.64"}, + {"description": "Committed spend", "amount": "-1.23"} + ], + "totalAmount": "3178.00" + } + """, + 200); + } + + @Test + void testInvalidRequests() { + ContainerTester tester = new ContainerTester(container, responseFiles); + assertEquals(SystemName.Public, tester.controller().system()); + + tester.assertJsonResponse(request("/pricing/v1/pricing"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No price information found in query\"}", + 400); + tester.assertJsonResponse(request("/pricing/v1/pricing?"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Error in query parameter, expected '=' between key and value: ''\"}", + 400); + tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No application resources found in query\"}", + 400); + tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0&resources"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Error in query parameter, expected '=' between key and value: 'resources'\"}", + 400); + tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0&resources="), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Error in query parameter, expected '=' between key and value: 'resources='\"}", + 400); + tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0&key=value"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Unknown query parameter 'key'\"}", + 400); + tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0&resources=key%3Dvalue"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Unknown resource type 'key'\"}", + 400); } /** * 2 clusters, with each having 1 node, with 1 vcpu, 1 Gb memory, 10 Gb disk and no GPU * price will be 20000 + 2000 + 200 */ - String urlEncodedPriceInformation() { + String urlEncodedPriceInformationLegacy(PricingInfo.SupportLevel supportLevel, boolean enclave) { String resources = URLEncoder.encode("nodes=1,vcpu=1,memoryGb=1,diskGb=10,gpuMemoryGb=0", UTF_8); - return "supportLevel=basic&committedSpend=0&enclave=false" + + return "supportLevel=" + supportLevel.name().toLowerCase() + "&committedSpend=100&enclave=" + enclave + "&resources=" + resources + "&resources=" + resources; } - String urlEncodedPriceInformationWithMissingValueInResourcs() { - return URLEncoder.encode("supportLevel=basic&committedSpend=0&enclave=false&resources", UTF_8); + /** + * 1 app, with 2 clusters (with total resources for all clusters with each having + * 1 node, with 1 vcpu, 1 Gb memory, 10 Gb disk and no GPU, + * price will be 20000 + 2000 + 200 + */ + String urlEncodedPriceInformation1App(PricingInfo.SupportLevel supportLevel) { + return "application=" + URLEncoder.encode("name=myapp,vcpu=2,memoryGb=2,diskGb=20,gpuMemoryGb=0", UTF_8) + + "&supportLevel=" + supportLevel.name().toLowerCase() + "&committedSpend=100"; + } + + /** + * 1 app, with 2 clusters (with total resources for all clusters with each having + * 1 node, with 1 vcpu, 1 Gb memory, 10 Gb disk and no GPU, + * price will be 20000 + 2000 + 200 + */ + String urlEncodedPriceInformation1AppEnclave(PricingInfo.SupportLevel supportLevel) { + return "application=" + URLEncoder.encode("name=myapp,enclaveVcpu=2,enclaveMemoryGb=2,enclaveDiskGb=20,enclaveGpuMemoryGb=0", UTF_8) + + "&supportLevel=" + supportLevel.name().toLowerCase() + "&committedSpend=100"; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 2d9c2f40a2a..a10bfd46b0c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -1068,11 +1068,9 @@ public class RoutingPoliciesTest { } @Test - public void generated_endpoints() { - var tester = new RoutingPoliciesTester(SystemName.Public); + public void combined_endpoint_config() { + var tester = new RoutingPoliciesTester(SystemName.Public).setEndpointConfig(EndpointConfig.combined); var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); - addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); // Deploy application int clustersPerZone = 2; @@ -1093,10 +1091,10 @@ public class RoutingPoliciesTest { // Deployment creates generated zone names List<String> expectedRecords = List.of( // save me, jebus! - "a6414896.cafed00d.aws-eu-west-1.w.vespa-app.cloud", - "b36bf591.cafed00d.z.vespa-app.cloud", + "a6414896.f5549014.aws-eu-west-1.w.vespa-app.cloud", + "aa7591aa.f5549014.z.vespa-app.cloud", "bar.app1.tenant1.a.vespa-app.cloud", - "bc50b636.cafed00d.z.vespa-app.cloud", + "bc50b636.f5549014.z.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1.w.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1.w.vespa-app.cloud", @@ -1105,16 +1103,16 @@ public class RoutingPoliciesTest { "c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", - "c33db5ed.cafed00d.z.vespa-app.cloud", - "d467800f.cafed00d.z.vespa-app.cloud", - "d71005bf.cafed00d.z.vespa-app.cloud", - "dd0971b4.cafed00d.z.vespa-app.cloud", - "eb48ad53.cafed00d.z.vespa-app.cloud", - "ec1e1288.cafed00d.z.vespa-app.cloud", - "f2fa41ec.cafed00d.g.vespa-app.cloud", - "f411d177.cafed00d.z.vespa-app.cloud", - "f4a4d111.cafed00d.a.vespa-app.cloud", - "fcf1bd63.cafed00d.aws-us-east-1.w.vespa-app.cloud", + "c33db5ed.f5549014.z.vespa-app.cloud", + "d467800f.f5549014.z.vespa-app.cloud", + "d71005bf.f5549014.z.vespa-app.cloud", + "dd0971b4.f5549014.g.vespa-app.cloud", + "eb48ad53.f5549014.z.vespa-app.cloud", + "ec1e1288.f5549014.z.vespa-app.cloud", + "f2fa41ec.f5549014.a.vespa-app.cloud", + "f411d177.f5549014.z.vespa-app.cloud", + "f4a4d111.f5549014.z.vespa-app.cloud", + "fcf1bd63.f5549014.aws-us-east-1.w.vespa-app.cloud", "foo.app1.tenant1.g.vespa-app.cloud" ); assertEquals(expectedRecords, tester.recordNames()); @@ -1178,23 +1176,23 @@ public class RoutingPoliciesTest { .build(); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(List.of( - "b36bf591.cafed00d.z.vespa-app.cloud", + "aa7591aa.f5549014.z.vespa-app.cloud", "bar.app1.tenant1.a.vespa-app.cloud", - "bc50b636.cafed00d.z.vespa-app.cloud", + "bc50b636.f5549014.z.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", - "c33db5ed.cafed00d.z.vespa-app.cloud", - "d467800f.cafed00d.z.vespa-app.cloud", - "d71005bf.cafed00d.z.vespa-app.cloud", - "dd0971b4.cafed00d.z.vespa-app.cloud", - "eb48ad53.cafed00d.z.vespa-app.cloud", - "ec1e1288.cafed00d.z.vespa-app.cloud", - "f411d177.cafed00d.z.vespa-app.cloud", - "f4a4d111.cafed00d.a.vespa-app.cloud" + "c33db5ed.f5549014.z.vespa-app.cloud", + "d467800f.f5549014.z.vespa-app.cloud", + "d71005bf.f5549014.z.vespa-app.cloud", + "eb48ad53.f5549014.z.vespa-app.cloud", + "ec1e1288.f5549014.z.vespa-app.cloud", + "f2fa41ec.f5549014.a.vespa-app.cloud", + "f411d177.f5549014.z.vespa-app.cloud", + "f4a4d111.f5549014.z.vespa-app.cloud" ), tester.recordNames()); // Removing application removes all records @@ -1206,11 +1204,9 @@ public class RoutingPoliciesTest { } @Test - public void generated_endpoints_enable_token() { - var tester = new RoutingPoliciesTester(SystemName.Public); + public void generated_endpoint_config_with_token() { + var tester = new RoutingPoliciesTester(SystemName.Public).setEndpointConfig(EndpointConfig.generated); var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); - tester.controllerTester().flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); // Deploy application without token @@ -1270,12 +1266,9 @@ public class RoutingPoliciesTest { } @Test - public void generated_endpoints_only() { - var tester = new RoutingPoliciesTester(SystemName.Public); + public void generated_endpoint_config() { + var tester = new RoutingPoliciesTester(SystemName.Public).setEndpointConfig(EndpointConfig.generated); var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.controllerTester().flagSource() - .withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true) - .withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); // Deploy application @@ -1317,12 +1310,10 @@ public class RoutingPoliciesTest { } @Test - public void generated_endpoints_multi_instance() { - var tester = new RoutingPoliciesTester(SystemName.Public); + public void combined_endpoint_config_with_multiple_instances() { + var tester = new RoutingPoliciesTester(SystemName.Public).setEndpointConfig(EndpointConfig.combined); var context0 = tester.newDeploymentContext("tenant1", "app1", "default"); var context1 = tester.newDeploymentContext("tenant1", "app1", "beta"); - tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); - addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); // Deploy application int clustersPerZone = 1; @@ -1338,11 +1329,11 @@ public class RoutingPoliciesTest { tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1); context0.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(List.of("a0.app1.tenant1.a.vespa-app.cloud", - "a9c8c045.cafed00d.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c0.beta.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", - "e144a11b.cafed00d.z.vespa-app.cloud", - "ee82b867.cafed00d.a.vespa-app.cloud"), + "cbff1506.f5549014.z.vespa-app.cloud", + "e144a11b.f5549014.a.vespa-app.cloud", + "ee82b867.f5549014.z.vespa-app.cloud"), tester.recordNames()); tester.assertTargets(context0.application().id(), EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(context0.deploymentIdIn(zone1), 1, context1.deploymentIdIn(zone1), 1)); @@ -1356,11 +1347,11 @@ public class RoutingPoliciesTest { .build(); context0.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(List.of("a0.app1.tenant1.a.vespa-app.cloud", - "a9c8c045.cafed00d.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c0.beta.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", - "e144a11b.cafed00d.z.vespa-app.cloud", - "ee82b867.cafed00d.a.vespa-app.cloud"), + "cbff1506.f5549014.z.vespa-app.cloud", + "e144a11b.f5549014.a.vespa-app.cloud", + "ee82b867.f5549014.z.vespa-app.cloud"), tester.recordNames()); tester.assertTargets(context0.application().id(), EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(context1.deploymentIdIn(zone1), 1)); @@ -1374,10 +1365,9 @@ public class RoutingPoliciesTest { } @Test - public void generated_endpoint_migration_with_global_endpoint() { - var tester = new RoutingPoliciesTester(SystemName.Public); + public void migrate_legacy_to_combined_endpoint_config_with_global_endpoint() { + var tester = new RoutingPoliciesTester(SystemName.Public).setEndpointConfig(EndpointConfig.legacy); var context = tester.newDeploymentContext("tenant1", "app1", "default"); - addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); // Deploy application int clustersPerZone = 2; @@ -1392,8 +1382,8 @@ public class RoutingPoliciesTest { context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.of("foo"), 0, zone1, zone2); - // Switch to generated - tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + // Switch to combined + tester.setEndpointConfig(EndpointConfig.combined); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instance().id(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"), 0, Map.of(zone1, 1L, zone2, 1L), true); @@ -1403,9 +1393,13 @@ public class RoutingPoliciesTest { EndpointCertificate cert = new EndpointCertificate("testKey", "testCert", 1, 0, "request-id", Optional.of("leaf-request-uuid"), - List.of("name1", "name2"), - "", Optional.empty(), - Optional.empty(), Optional.of(id)); + List.of("*." + id + ".z.vespa-app.cloud", + "*." + id + ".g.vespa-app.cloud", + "*." + id + ".a.vespa-app.cloud"), + "", + Optional.empty(), + Optional.empty(), + Optional.of(id)); UnassignedCertificate pooledCert = new UnassignedCertificate(cert, state); tester.controllerTester().controller().curator().writeUnassignedCertificate(pooledCert); } @@ -1521,6 +1515,12 @@ public class RoutingPoliciesTest { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } + public RoutingPoliciesTester setEndpointConfig(EndpointConfig config) { + tester.controllerTester().flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), config.supportsLegacy()); + tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), config.supportsGenerated()); + return this; + } + public RoutingPolicies routingPolicies() { return tester.controllerTester().controller().routing().policies(); } diff --git a/controller-server/src/test/resources/mail/notification.txt b/controller-server/src/test/resources/mail/notification.html index 35db37fbc12..c8d0037426b 100644 --- a/controller-server/src/test/resources/mail/notification.txt +++ b/controller-server/src/test/resources/mail/notification.html @@ -383,11 +383,12 @@ style="vertical-align: top" width="100%" > - <tbody> - <tr> - <td - align="left" - style=" + +<tbody> +<tr> + <td + align="left" + style=" font-size: 0px; padding: 0px 25px 0px 25px; padding-top: 0px; @@ -396,9 +397,9 @@ padding-left: 50px; word-break: break-word; " - > - <div - style=" + > + <div + style=" font-family: Open Sans, Helvetica, Arial, sans-serif; font-size: 13px; @@ -406,23 +407,23 @@ text-align: left; color: #797e82; " - > - <h1 - style=" + > + <h1 + style=" text-align: center; color: #000000; line-height: 32px; " - > - Vespa Cloud Notifications - </h1> - </div> - </td> - </tr> - <tr> - <td - align="left" - style=" + > + Vespa Cloud Notifications + </h1> + </div> + </td> +</tr> +<tr> + <td + align="left" + style=" font-size: 0px; padding: 0px 25px 0px 25px; padding-top: 0px; @@ -431,9 +432,9 @@ padding-left: 50px; word-break: break-word; " - > - <div - style=" + > + <div + style=" font-family: Open Sans, Helvetica, Arial, sans-serif; font-size: 13px; @@ -441,51 +442,54 @@ text-align: left; color: #797e82; " - > - <p> - There are problems with tests for default.default: - </p> - <p>Test package has production tests, but no production tests are declared in deployment.xml</p><p>See <a href="https://docs.vespa.ai/en/testing.html">https://docs.vespa.ai/en/testing.html</a> for details on how to write system tests for Vespa</p> - </div> - </td> - </tr> - <tr> - <td - align="center" - vertical-align="middle" - style=" + > + +<p> + There are problems with tests for default.default: +</p> +<p>Test package has production tests, but no production tests are declared in deployment.xml</p> +<p>See <a href="https://docs.vespa.ai/en/testing.html">https://docs.vespa.ai/en/testing.html</a> for details on how to write system tests for Vespa</p> + + </div> + </td> +</tr> +<tr> + <td + align="center" + vertical-align="middle" + style=" font-size: 0px; padding: 10px 25px; padding-top: 20px; padding-bottom: 20px; word-break: break-word; " - > - <table - border="0" - cellpadding="0" - cellspacing="0" - role="presentation" - style="border-collapse: separate; line-height: 100%" - > - <tbody> - <tr> - <td - align="center" - bgcolor="#005A8E" - role="presentation" - style=" + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="border-collapse: separate; line-height: 100%" + > + <tbody> + <tr> + <td + align="center" + bgcolor="#005A8E" + role="presentation" + style=" border: none; border-radius: 100px; cursor: auto; mso-padding-alt: 15px 25px 15px 25px; background: #005a8e; " - valign="middle" - > - <a - href="https://dashboard.tld/tenant/tenant1/application/default/prod/instance" - style=" + valign="middle" + > + <a + href="https://dashboard.tld/tenant/tenant1/application/default/prod/instance" + style=" display: inline-block; background: #005a8e; color: #ffffff; @@ -501,20 +505,20 @@ mso-padding-alt: 0px; border-radius: 100px; " - target="_blank" - ><b style="font-weight: 700" - ><b style="font-weight: 700" - >Go to Console</b - ></b - ></a - > - </td> - </tr> - </tbody> - </table> - </td> - </tr> - </tbody> + target="_blank" + ><b style="font-weight: 700" + ><b style="font-weight: 700" + >Go to Console</b + ></b + ></a + > + </td> + </tr> + </tbody> + </table> + </td> +</tr> +</tbody> </table> </div> <!--[if mso | IE]></td></tr></table><![endif]--> diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index c015e9a9a33..0b2acfe29ee 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -35,7 +35,7 @@ <aopalliance.vespa.version>1.0</aopalliance.vespa.version> <commons-logging.vespa.version>1.2</commons-logging.vespa.version> <!-- This version is exported by jdisc via jcl-over-slf4j. --> <error-prone-annotations.vespa.version>2.22.0</error-prone-annotations.vespa.version> - <guava.vespa.version>32.1.2-jre</guava.vespa.version> + <guava.vespa.version>32.1.3-jre</guava.vespa.version> <guice.vespa.version>6.0.0</guice.vespa.version> <jackson2.vespa.version>2.15.2</jackson2.vespa.version> <jackson-databind.vespa.version>2.15.2</jackson-databind.vespa.version> @@ -65,8 +65,8 @@ <assertj.vespa.version>3.24.2</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <athenz.vespa.version>1.11.42</athenz.vespa.version> - <aws-sdk.vespa.version>1.12.540</aws-sdk.vespa.version> + <athenz.vespa.version>1.11.43</athenz.vespa.version> + <aws-sdk.vespa.version>1.12.565</aws-sdk.vespa.version> <!-- Athenz END --> <!-- WARNING: If you change curator version, you also need to update @@ -78,8 +78,11 @@ <bouncycastle.vespa.version>1.76</bouncycastle.vespa.version> <byte-buddy.vespa.version>1.14.9</byte-buddy.vespa.version> <checker-qual.vespa.version>3.38.0</checker-qual.vespa.version> + <commons-beanutils.vespa.version>1.9.4</commons-beanutils.vespa.version> <commons-codec.vespa.version>1.16.0</commons-codec.vespa.version> + <commons-collections.vespa.version>3.2.2</commons-collections.vespa.version> <commons-csv.vespa.version>1.10.0</commons-csv.vespa.version> + <commons-digester.vespa.version>3.2</commons-digester.vespa.version> <commons-exec.vespa.version>1.3</commons-exec.vespa.version> <commons-io.vespa.version>2.14.0</commons-io.vespa.version> <commons-lang3.vespa.version>3.13.0</commons-lang3.vespa.version> @@ -126,6 +129,8 @@ <spifly.vespa.version>1.3.6</spifly.vespa.version> <snappy.vespa.version>1.1.10.5</snappy.vespa.version> <surefire.vespa.version>3.1.2</surefire.vespa.version> + <velocity.vespa.version>2.3</velocity.vespa.version> + <velocity.tools.vespa.version>3.1</velocity.tools.vespa.version> <wiremock.vespa.version>3.2.0</wiremock.vespa.version> <xerces.vespa.version>2.12.2</xerces.vespa.version> <zero-allocation-hashing.vespa.version>0.16</zero-allocation-hashing.vespa.version> diff --git a/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java index 6ef36f0013a..c2c22558a32 100644 --- a/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java +++ b/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java @@ -10,10 +10,10 @@ import java.util.ListIterator; import java.util.NoSuchElementException; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class ListAnnotationContainer extends IteratingAnnotationContainer { - private final List<Annotation> annotations = new LinkedList<Annotation>(); + private final List<Annotation> annotations = new LinkedList<>(); @Override void annotateAll(Collection<Annotation> annotations) { @@ -55,7 +55,7 @@ public class ListAnnotationContainer extends IteratingAnnotationContainer { private boolean nextCalled = false; AnnotationIterator(ListIterator<Annotation> baseIt, IdentityHashMap<SpanNode, SpanNode> nodes) { - this.base = new PeekableListIterator<Annotation>(baseIt); + this.base = new PeekableListIterator(baseIt); this.nodes = nodes; } diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanNodeParent.java b/document/src/main/java/com/yahoo/document/annotation/SpanNodeParent.java index 167ce4589da..a4d178e6925 100644 --- a/document/src/main/java/com/yahoo/document/annotation/SpanNodeParent.java +++ b/document/src/main/java/com/yahoo/document/annotation/SpanNodeParent.java @@ -6,7 +6,7 @@ import com.yahoo.document.datatypes.StringFieldValue; /** * An interface to be implemented by classes that can be parents of SpanNodes. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @see SpanNode#getParent() */ public interface SpanNodeParent { @@ -15,12 +15,12 @@ public interface SpanNodeParent { * * @return the SpanTree of this, if it belongs to a SpanTree, otherwise null. */ - public SpanTree getSpanTree(); + SpanTree getSpanTree(); /** * Returns the StringFieldValue that this node belongs to, if any. * * @return the StringFieldValue that this node belongs to, if any, otherwise null. */ - public StringFieldValue getStringFieldValue(); + StringFieldValue getStringFieldValue(); } diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanTree.java b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java index 3dc27171d8c..f785cf3b3ec 100644 --- a/document/src/main/java/com/yahoo/document/annotation/SpanTree.java +++ b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java @@ -22,7 +22,7 @@ import java.util.Map; /** * A SpanTree holds a root node of a tree of SpanNodes, and a List of Annotations pointing to these nodes - * or each other. It also has a name. + * or each other. It also has a name. * * @author Einar M R Rosenvinge * @see com.yahoo.document.annotation.SpanNode @@ -36,7 +36,7 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl private StringFieldValue stringFieldValue; /** - * WARNING! Only to be used by deserializers! Creates an empty SpanTree instance. + * WARNING! Only to be used by deserializers! Creates an empty SpanTree instance. */ public SpanTree() { } @@ -65,8 +65,8 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl public SpanTree(SpanTree otherToCopy) { name = otherToCopy.name; setRoot(copySpan(otherToCopy.root)); - List<Annotation> annotationsToCopy = new ArrayList<Annotation>(otherToCopy.getAnnotations()); - List<Annotation> newAnnotations = new ArrayList<Annotation>(annotationsToCopy.size()); + List<Annotation> annotationsToCopy = new ArrayList<>(otherToCopy.getAnnotations()); + List<Annotation> newAnnotations = new ArrayList<>(annotationsToCopy.size()); for (Annotation otherAnnotationToCopy : annotationsToCopy) { newAnnotations.add(new Annotation(otherAnnotationToCopy)); @@ -153,7 +153,7 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl } private IdentityHashMap<Annotation, Integer> getAnnotations(List<Annotation> annotationsToCopy) { - IdentityHashMap<Annotation, Integer> map = new IdentityHashMap<Annotation, Integer>(); + IdentityHashMap<Annotation, Integer> map = new IdentityHashMap<>(); for (int i = 0; i < annotationsToCopy.size(); i++) { map.put(annotationsToCopy.get(i), i); } @@ -162,7 +162,7 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl private List<SpanNode> getSpanNodes() { - ArrayList<SpanNode> nodes = new ArrayList<SpanNode>(); + ArrayList<SpanNode> nodes = new ArrayList<>(); nodes.add(root); Iterator<SpanNode> it = root.childIteratorRecursive(); while (it.hasNext()) { @@ -172,7 +172,7 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl } private static IdentityHashMap<SpanNode, Integer> getSpanNodes(SpanTree otherToCopy) { - IdentityHashMap<SpanNode, Integer> map = new IdentityHashMap<SpanNode, Integer>(); + IdentityHashMap<SpanNode, Integer> map = new IdentityHashMap<>(); int spanNodeCounter = 0; map.put(otherToCopy.getRoot(), spanNodeCounter++); Iterator<SpanNode> it = otherToCopy.getRoot().childIteratorRecursive(); diff --git a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java index 797d89226f3..8b4b94f6bbf 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java @@ -106,7 +106,7 @@ public class StringFieldValue extends FieldValue { } /** - * Sets a new value for this StringFieldValue. NOTE that doing so will clear all span trees from this value, + * Sets a new value for this StringFieldValue. NOTE that doing so will clear all span trees from this value, * since they most certainly will not make sense for a new string value. * * @param o the new String to assign to this. An argument of null is equal to calling clear(). diff --git a/document/src/test/java/com/yahoo/document/datatypes/ArrayTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/ArrayTestCase.java index 0554b7349c2..7b1e161ef0e 100755 --- a/document/src/test/java/com/yahoo/document/datatypes/ArrayTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/ArrayTestCase.java @@ -77,7 +77,7 @@ public class ArrayTestCase { @Test public void testWrappedList() { - Array<StringFieldValue> array = new Array<StringFieldValue>(DataType.getArray(DataType.STRING)); + Array<StringFieldValue> array = new Array<>(DataType.getArray(DataType.STRING)); List<String> list = new ArrayList<>(); list.add("foo"); list.add("bar"); @@ -217,10 +217,10 @@ public class ArrayTestCase { assertEquals(new StringFieldValue("apple"), subArray.get(1)); - assertEquals(false, array.containsAll(Arrays.<StringFieldValue>asList(new StringFieldValue("bob")))); - assertEquals(true, array.containsAll(Arrays.<StringFieldValue>asList(new StringFieldValue("foo"), new StringFieldValue("boo"), new StringFieldValue("apple")))); + assertEquals(false, array.containsAll(List.of(new StringFieldValue("bob")))); + assertEquals(true, array.containsAll(List.of(new StringFieldValue("foo"), new StringFieldValue("boo"), new StringFieldValue("apple")))); - array.removeAll(Arrays.<StringFieldValue>asList(new StringFieldValue("foo"), new StringFieldValue("boo"))); + array.removeAll(List.of(new StringFieldValue("foo"), new StringFieldValue("boo"))); assertEquals(1, array.size()); assertEquals(1, list.size()); @@ -249,7 +249,7 @@ public class ArrayTestCase { assertFalse(it.hasNext()); } - array.addAll(Arrays.<StringFieldValue>asList(new StringFieldValue("microsoft"), new StringFieldValue("google"))); + array.addAll(List.of(new StringFieldValue("microsoft"), new StringFieldValue("google"))); assertEquals(4, array.size()); assertEquals(4, list.size()); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AdaptiveLoadBalancer.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AdaptiveLoadBalancer.java index 4a4cf0fd5c8..ae934857e2c 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AdaptiveLoadBalancer.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/AdaptiveLoadBalancer.java @@ -7,7 +7,7 @@ import java.util.List; import java.util.Random; /** - * Will pick 2 random candidates and select the one with least pending operations. + * Will pick 2 random candidates and select the one with the least pending operations. * * @author baldersheim */ diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java index ddd04a3ca53..4f8227b35a0 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java @@ -23,7 +23,7 @@ public class LocalServicePolicy implements DocumentProtocolRoutingPolicy { private final Map<String, CacheEntry> cache = new HashMap<>(); /** - * Constructs a policy that will choose local services that match the slobrok pattern in which this policy occured. + * Constructs a policy that will choose local services that match the slobrok pattern in which this policy occurred. * If no local service can be found, this policy simply returns the asterisk to allow the network to choose any. * * @param param The address to use for this, if empty this will resolve to hostname. diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 8c49ec9ba29..cae11d66d13 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -79,6 +79,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; /** @@ -500,6 +501,29 @@ public class DocumentGenPluginTest { assertEquals(book.getFieldValue("isbn"), new StringFieldValue("ISBN YEP")); } + @Test + public void testSetterValidation() { + Book book = new Book(new DocumentId("id:book:book::0")); + + book.setAuthor("Herman Melville"); + assertEquals("The string field value contains illegal code point 0x16", + assertThrows(IllegalArgumentException.class, + () -> book.setAuthor("He\u0016rman Malville")).getMessage()); + + book.setRef(new DocumentId("id:ns:parent::foo")); + assertEquals("Can't assign document ID 'id:ns:common::bar' (of type 'common') to reference of document type 'parent'", + assertThrows(IllegalArgumentException.class, + () -> book.setRef(new DocumentId("id:ns:common::bar"))).getMessage()); + + book.setStringmap(Map.of("foo", "bar")); + assertEquals("The string field value contains illegal code point 0x16", + assertThrows(IllegalArgumentException.class, + () -> book.setStringmap(Map.of("foo", "bar\u0016"))).getMessage()); + assertEquals("The string field value contains illegal code point 0x16", + assertThrows(IllegalArgumentException.class, + () -> book.setStringmap(Map.of("bar\u0016", "foo"))).getMessage()); + } + public static class BookProcessor extends DocumentProcessor { public Progress process(Processing processing) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index e8b0ab1f1d7..56cd06d3b35 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -429,6 +429,13 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); + public static final UnboundStringFlag ENDPOINT_CONFIG = defineStringFlag( + "endpoint-config", "legacy", + List.of("mpolden", "tokle"), "2023-10-06", "2024-02-01", + "Set the endpoint config to use for an application. Must be 'legacy', 'combined' or 'generated'. See EndpointConfig for further details", + "Takes effect on next deployment through controller", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureConjunction.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureConjunction.java index 3556fb0a739..8b0cd62df07 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureConjunction.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureConjunction.java @@ -13,7 +13,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; /** - * <p>This class implements a Future<Boolean> that is conjunction of zero or more other Future<Boolean>s, + * <p>This class implements a Future<Boolean> that is a conjunction of zero or more other Future<Boolean>s, * i.e. it evaluates to <code>true</code> if, and only if, all its operands evaluate to <code>true</code>. To use this class, * simply create an instance of it and add operands to it using the {@link #addOperand(CompletableFuture)} method.</p> * diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java index 28037ce6fb4..b60b62e3f86 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java @@ -119,7 +119,7 @@ public abstract class RequestDispatch implements Future<Response>, ResponseHandl try { return futureResponse.get(); } catch (InterruptedException | ExecutionException e) { - throw new IllegalStateException(e); // Should not happens since both futures are complete + throw new IllegalStateException(e); // Should not happen since both futures are complete } }); } diff --git a/jdisc_core/src/test/resources/exportPackages.properties b/jdisc_core/src/test/resources/exportPackages.properties index 5726cf8d924..dc6e4ab0962 100644 --- a/jdisc_core/src/test/resources/exportPackages.properties +++ b/jdisc_core/src/test/resources/exportPackages.properties @@ -1,3 +1,3 @@ #generated by com.yahoo.jdisc.core.ExportPackages #Fri Jul 07 16:04:11 CEST 2023 -exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", java.util.jar; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.security.auth.callback; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="32.1.2",com.google.common.base;version\="32.1.2";uses\:\="javax.annotation",com.google.common.cache;version\="32.1.2";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent,javax.annotation",com.google.common.collect;version\="32.1.2";uses\:\="com.google.common.base,javax.annotation",com.google.common.escape;version\="32.1.2";uses\:\="com.google.common.base,javax.annotation",com.google.common.eventbus;version\="32.1.2",com.google.common.graph;version\="32.1.2";uses\:\="com.google.common.collect,javax.annotation",com.google.common.hash;version\="32.1.2";uses\:\="com.google.common.base,javax.annotation",com.google.common.html;version\="32.1.2";uses\:\="com.google.common.escape",com.google.common.io;version\="32.1.2";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash,javax.annotation",com.google.common.math;version\="32.1.2";uses\:\="javax.annotation",com.google.common.net;version\="32.1.2";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape,javax.annotation",com.google.common.primitives;version\="32.1.2";uses\:\="com.google.common.base,javax.annotation",com.google.common.reflect;version\="32.1.2";uses\:\="com.google.common.collect,com.google.common.io,javax.annotation",com.google.common.util.concurrent;version\="32.1.2";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal,javax.annotation",com.google.common.xml;version\="32.1.2";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, com.sun.jna;version\=5.11.0, com.sun.jna.ptr;version\=5.11.0, com.sun.jna.win32;version\=5.11.0, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.llm.client.openai;version\=1.0.0,ai.vespa.llm.completion;version\=1.0.0,ai.vespa.llm.test;version\=1.0.0,ai.vespa.llm;version\=1.0.0,ai.vespa.net;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.helpers;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" +exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", java.util.jar; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.security.auth.callback; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="32.1.3",com.google.common.base;version\="32.1.3";uses\:\="javax.annotation",com.google.common.cache;version\="32.1.3";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent,javax.annotation",com.google.common.collect;version\="32.1.3";uses\:\="com.google.common.base,javax.annotation",com.google.common.escape;version\="32.1.3";uses\:\="com.google.common.base,javax.annotation",com.google.common.eventbus;version\="32.1.3",com.google.common.graph;version\="32.1.3";uses\:\="com.google.common.collect,javax.annotation",com.google.common.hash;version\="32.1.3";uses\:\="com.google.common.base,javax.annotation",com.google.common.html;version\="32.1.3";uses\:\="com.google.common.escape",com.google.common.io;version\="32.1.3";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash,javax.annotation",com.google.common.math;version\="32.1.3";uses\:\="javax.annotation",com.google.common.net;version\="32.1.3";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape,javax.annotation",com.google.common.primitives;version\="32.1.3";uses\:\="com.google.common.base,javax.annotation",com.google.common.reflect;version\="32.1.3";uses\:\="com.google.common.collect,com.google.common.io,javax.annotation",com.google.common.util.concurrent;version\="32.1.3";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal,javax.annotation",com.google.common.xml;version\="32.1.3";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, com.sun.jna;version\=5.11.0, com.sun.jna.ptr;version\=5.11.0, com.sun.jna.win32;version\=5.11.0, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.llm.client.openai;version\=1.0.0,ai.vespa.llm.completion;version\=1.0.0,ai.vespa.llm.test;version\=1.0.0,ai.vespa.llm;version\=1.0.0,ai.vespa.net;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.helpers;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" diff --git a/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java b/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java index 8fab523d4ae..88e3e1a89bc 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/ErrorCode.java @@ -20,7 +20,7 @@ public final class ErrorCode { /** No addresses found for the services of the message route. */ public static final int NO_ADDRESS_FOR_SERVICE = TRANSIENT_ERROR + 2; - /** A connection problem occured while sending. */ + /** A connection problem occurred while sending. */ public static final int CONNECTION_ERROR = TRANSIENT_ERROR + 3; /** The session specified for the message is unknown. */ @@ -50,10 +50,10 @@ public final class ErrorCode { /** No services found for the message route. */ public static final int NO_SERVICES_FOR_ROUTE = FATAL_ERROR + 3; - /** An error occured while encoding the message. */ + /** An error occurred while encoding the message. */ public static final int ENCODE_ERROR = FATAL_ERROR + 5; - /** A fatal network error occured while sending. */ + /** A fatal network error occurred while sending. */ public static final int NETWORK_ERROR = FATAL_ERROR + 6; /** The protocol specified for the message is unknown. */ @@ -77,7 +77,7 @@ public final class ErrorCode { /** Exception thrown by routing policy. */ public static final int POLICY_ERROR = FATAL_ERROR + 13; - /** An error occured while sequencing a message. */ + /** An error occurred while sequencing a message. */ public static final int SEQUENCE_ERROR = FATAL_ERROR + 14; /** An application specific non-recoverable error. */ diff --git a/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java b/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java index cbaeb25af58..7e8286d8793 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/IntermediateSession.java @@ -119,6 +119,7 @@ public final class IntermediateSession implements MessageHandler, ReplyHandler, mbus.connect(name, broadcastName); } - @Override public void disconnect() { close(); } + @Override + public void disconnect() { close(); } } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/NetworkMultiplexer.java b/messagebus/src/main/java/com/yahoo/messagebus/network/NetworkMultiplexer.java index 0759a5661be..11d7a69313e 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/NetworkMultiplexer.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/NetworkMultiplexer.java @@ -6,6 +6,7 @@ import com.yahoo.messagebus.Protocol; import com.yahoo.text.Utf8Array; import java.util.Deque; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingContext.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingContext.java index 227dd546ad8..18b5de34bb4 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingContext.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingContext.java @@ -19,7 +19,7 @@ public class RoutingContext { private final RoutingNode node; private final int directive; - private final Set<Integer> consumableErrors = new HashSet<Integer>(); + private final Set<Integer> consumableErrors = new HashSet<>(); private boolean selectOnRetry = true; private Object context = null; diff --git a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java index 3676be90cd4..f03c54aa822 100644 --- a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java @@ -9,6 +9,8 @@ public enum ControllerMetrics implements VespaMetrics { ATHENZ_REQUEST_ERROR("athenz.request.error", Unit.REQUEST, "Controller: Athenz request error"), ARCHIVE_BUCKET_COUNT("archive.bucketCount", Unit.BUCKET, "Controller: Archive bucket count"), + DEPLOYMENT_JOBS_QUEUED("deployment.jobsQueued", Unit.TASK, "The number of deployment jobs queued"), + DEPLOYMENT_JOBS_ACTIVE("deployment.jobsActive", Unit.TASK, "The number of deployment jobs active"), DEPLOYMENT_START("deployment.start", Unit.DEPLOYMENT, "The number of started deployment jobs"), DEPLOYMENT_NODE_ALLOCATION_FAILURE("deployment.nodeAllocationFailure", Unit.DEPLOYMENT, "The number of deployments failed due to node allocation failures"), DEPLOYMENT_ENDPOINT_CERTIFICATE_TIMEOUT("deployment.endpointCertificateTimeout", Unit.DEPLOYMENT, "The number of deployments failed due to timeout acquiring endpoint certificate"), diff --git a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java index 6bffddb885a..9443a08e28b 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java @@ -143,6 +143,8 @@ public class InfrastructureMetricSet { addMetric(metrics, ControllerMetrics.ARCHIVE_BUCKET_COUNT.max()); addMetric(metrics, ControllerMetrics.BILLING_TENANTS.max()); + addMetric(metrics, ControllerMetrics.DEPLOYMENT_JOBS_QUEUED, EnumSet.of(count, sum)); + addMetric(metrics, ControllerMetrics.DEPLOYMENT_JOBS_ACTIVE, EnumSet.of(count, sum)); addMetric(metrics, ControllerMetrics.DEPLOYMENT_ABORT.count()); addMetric(metrics, ControllerMetrics.DEPLOYMENT_AVERAGE_DURATION.max()); addMetric(metrics, ControllerMetrics.DEPLOYMENT_CONVERGENCE_FAILURE.count()); diff --git a/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java index 6f79c1f20e3..4174aa6cb53 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java @@ -178,7 +178,7 @@ public class Vespa9VespaMetricSet { addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS_TOTAL.sum()); - addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_IS_ACTIVE.max()); + addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_IS_ACTIVE, EnumSet.of(min, max)); addMetric(metrics, ContainerMetrics.ATHENZ_TENANT_CERT_EXPIRY_SECONDS.min()); addMetric(metrics, ContainerMetrics.CONTAINER_IAM_ROLE_EXPIRY_SECONDS.baseName()); diff --git a/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java index 6f74f4327a5..a6ac18cf011 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java @@ -194,7 +194,7 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS_TOTAL, EnumSet.of(sum, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS_WITH_RETAINED_REFS.last()); - addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_IS_ACTIVE, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_IS_ACTIVE, EnumSet.of(min, max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_ACTIVATION_COUNT.last()); addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_ACTIVATION_FAILURE_COUNT.last()); addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_ACTIVATION_MILLIS.last()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 32f6ff32319..4fc20eca41e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -56,6 +56,7 @@ public final class Node implements Nodelike { private final Optional<String> modelName; private final Optional<TenantName> reservedTo; private final Optional<ApplicationId> exclusiveToApplicationId; + private final Optional<ApplicationId> provisionedForApplicationId; private final Optional<Duration> hostTTL; private final Optional<Instant> hostEmptyAt; private final Optional<ClusterSpec.Type> exclusiveToClusterType; @@ -93,9 +94,9 @@ public final class Node implements Nodelike { public Node(String id, Optional<String> extraId, IP.Config ipConfig, String hostname, Optional<String> parentHostname, Flavor flavor, Status status, State state, Optional<Allocation> allocation, History history, NodeType type, Reports reports, Optional<String> modelName, Optional<TenantName> reservedTo, - Optional<ApplicationId> exclusiveToApplicationId, Optional<Duration> hostTTL, Optional<Instant> hostEmptyAt, - Optional<ClusterSpec.Type> exclusiveToClusterType, Optional<String> switchHostname, - List<TrustStoreItem> trustStoreItems, CloudAccount cloudAccount, + Optional<ApplicationId> exclusiveToApplicationId, Optional<ApplicationId> provisionedForApplicationId, + Optional<Duration> hostTTL, Optional<Instant> hostEmptyAt, Optional<ClusterSpec.Type> exclusiveToClusterType, + Optional<String> switchHostname, List<TrustStoreItem> trustStoreItems, CloudAccount cloudAccount, Optional<WireguardKeyWithTimestamp> wireguardPubKey) { this.id = Objects.requireNonNull(id, "A node must have an ID"); this.extraId = Objects.requireNonNull(extraId, "Extra ID cannot be null"); @@ -112,6 +113,7 @@ public final class Node implements Nodelike { this.modelName = Objects.requireNonNull(modelName, "A null modelName is not permitted"); this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null"); this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId cannot be null"); + this.provisionedForApplicationId = Objects.requireNonNull(provisionedForApplicationId, "provisionedForApplicationId cannot be null"); this.hostTTL = Objects.requireNonNull(hostTTL, "hostTTL cannot be null"); this.hostEmptyAt = Objects.requireNonNull(hostEmptyAt, "hostEmptyAt cannot be null"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType cannot be null"); @@ -141,6 +143,9 @@ public final class Node implements Nodelike { if (type != NodeType.host && exclusiveToApplicationId.isPresent()) throw new IllegalArgumentException("Only tenant hosts can be exclusive to an application"); + if (provisionedForApplicationId.isPresent() && ! exclusiveToApplicationId.equals(provisionedForApplicationId)) + throw new IllegalArgumentException("exclusiveToApplicationId must be the same as provisionedForApplicationId when this is set"); + if (type != NodeType.host && hostTTL.isPresent()) throw new IllegalArgumentException("Only tenant hosts can have a TTL"); @@ -221,12 +226,21 @@ public final class Node implements Nodelike { /** * Returns the application this host is exclusive to, if any. Only tenant hosts can be exclusive to an application. - * If this is set, resources on this host cannot be allocated to any other application. This is set during - * provisioning and applies for the entire lifetime of the host + * If this is set, resources on this host cannot be allocated to any other application. Additionally, the host will + * not be reused once its allocated containers are deleted, i.e., this property can only be set <em>once</em> per host. */ public Optional<ApplicationId> exclusiveToApplicationId() { return exclusiveToApplicationId; } /** + * Returns the application this host was provisioned specifically for, if any. Only tenant hosts can be exclusive + * to an application. This property, when set, also implies {@link #exclusiveToApplicationId()}. + * This is set during provisioning and applies for the entire lifetime of the host. Provisioning a host specifically + * for an application allows access to application-specific resources, through integration with cloud providers' + * provisioning-with-secrets mechanisms. + */ + public Optional<ApplicationId> provisionedForApplicationId() { return provisionedForApplicationId; } + + /** * Returns the additional time to live of tenant host, in a dynamically provisioned zone, after all its child * nodes are removed, before being deprovisioned, if any. * This is set during provisioning and applies for the entire lifetime of the host. @@ -359,14 +373,14 @@ public final class Node implements Nodelike { /** Returns a node with the status assigned to the given value */ public Node with(Status status) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, - reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, - reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } @@ -375,35 +389,35 @@ public final class Node implements Nodelike { if (flavor.equals(this.flavor)) return this; History updateHistory = history.with(new History.Event(History.Event.Type.resized, agent, instant)); return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, updateHistory, type, - reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with the reboot generation set to generation */ public Node withReboot(Generation generation) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, allocation, - history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + history, type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with given id set */ public Node withId(String id) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, - history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + history, type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with model name set to given value */ public Node withModelName(String modelName) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, Optional.of(modelName), reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, Optional.of(modelName), reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with model name cleared */ public Node withoutModelName() { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, Optional.empty(), reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, Optional.empty(), reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } @@ -445,21 +459,21 @@ public final class Node implements Nodelike { */ public Node with(Allocation allocation) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, Optional.of(allocation), history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node with IP config set to the given value. */ public Node with(IP.Config ipConfig) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { return new Node(id, extraId, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state, allocation, - history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + history, type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } @@ -467,50 +481,56 @@ public final class Node implements Nodelike { if (type != NodeType.host) throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type); return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, Optional.of(tenant), exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, Optional.of(tenant), exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node which is not reserved to a tenant */ public Node withoutReservedTo() { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, Optional.empty(), exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, Optional.empty(), exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExclusiveToApplicationId(ApplicationId exclusiveTo) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), provisionedForApplicationId.filter(__ -> exclusiveTo != null), hostTTL, hostEmptyAt, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); + } + + public Node withProvisionedForApplicationId(ApplicationId provisionedFor) { + return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, + type, reports, modelName, reservedTo, Optional.ofNullable(provisionedFor), Optional.ofNullable(provisionedFor), hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExtraId(Optional<String> extraId) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withHostTTL(Duration hostTTL) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, Optional.ofNullable(hostTTL), hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, Optional.ofNullable(hostTTL), hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withHostEmptyAt(Instant hostEmptyAt) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, Optional.ofNullable(hostEmptyAt), + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, Optional.ofNullable(hostEmptyAt), exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExclusiveToClusterType(ClusterSpec.Type exclusiveTo) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, Optional.ofNullable(exclusiveTo), switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withWireguardPubkey(WireguardKeyWithTimestamp wireguardPubkey) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, Optional.ofNullable(wireguardPubkey)); } @@ -518,7 +538,7 @@ public final class Node implements Nodelike { /** Returns a copy of this node with switch hostname set to given value */ public Node withSwitchHostname(String switchHostname) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, Optional.ofNullable(switchHostname), trustStoreItems, cloudAccount, wireguardPubKey); } @@ -572,19 +592,19 @@ public final class Node implements Nodelike { /** Returns a copy of this node with the given history. */ public Node with(History history) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node with(Reports reports) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node with(List<TrustStoreItem> trustStoreItems) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, exclusiveToApplicationId, provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } @@ -722,6 +742,7 @@ public final class Node implements Nodelike { private String modelName; private TenantName reservedTo; private ApplicationId exclusiveToApplicationId; + private ApplicationId provisionedForApplicationId; private Duration hostTTL; private Instant hostEmptyAt; private ClusterSpec.Type exclusiveToClusterType; @@ -763,6 +784,11 @@ public final class Node implements Nodelike { return this; } + public Builder provisionedForApplicationId(ApplicationId provisionedFor) { + this.provisionedForApplicationId = provisionedFor; + return exclusiveToApplicationId(provisionedFor); + } + public Builder hostTTL(Duration hostTTL) { this.hostTTL = hostTTL; return this; @@ -833,9 +859,9 @@ public final class Node implements Nodelike { flavor, Optional.ofNullable(status).orElseGet(Status::initial), state, Optional.ofNullable(allocation), Optional.ofNullable(history).orElseGet(History::empty), type, Optional.ofNullable(reports).orElseGet(Reports::new), Optional.ofNullable(modelName), Optional.ofNullable(reservedTo), Optional.ofNullable(exclusiveToApplicationId), - Optional.ofNullable(hostTTL), Optional.ofNullable(hostEmptyAt), Optional.ofNullable(exclusiveToClusterType), - Optional.ofNullable(switchHostname), Optional.ofNullable(trustStoreItems).orElseGet(List::of), cloudAccount, - Optional.ofNullable(wireguardPubKey)); + Optional.ofNullable(provisionedForApplicationId), Optional.ofNullable(hostTTL), Optional.ofNullable(hostEmptyAt), + Optional.ofNullable(exclusiveToClusterType), Optional.ofNullable(switchHostname), + Optional.ofNullable(trustStoreItems).orElseGet(List::of), cloudAccount, Optional.ofNullable(wireguardPubKey)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 99c69048af7..433a37e9686 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -137,11 +137,11 @@ public class NodeRepositoryMaintenance extends AbstractComponent { hostResumeProvisionerInterval = Duration.ofMinutes(3); diskReplacerInterval = Duration.ofMinutes(3); failedExpirerInterval = Duration.ofMinutes(10); - failGrace = Duration.ofMinutes(20); + failGrace = Duration.ofMinutes(10); infrastructureProvisionInterval = Duration.ofMinutes(3); loadBalancerExpirerInterval = Duration.ofMinutes(5); metricsInterval = Duration.ofMinutes(1); - nodeFailerInterval = Duration.ofMinutes(7); + nodeFailerInterval = Duration.ofMinutes(4); nodeFailureStatusUpdateInterval = Duration.ofMinutes(2); nodeMetricsCollectionInterval = Duration.ofMinutes(1); expeditedChangeRedeployInterval = Duration.ofMinutes(3); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index 976f3543298..4b81e580b64 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -221,7 +221,7 @@ public class CuratorDb { toState.isAllocated() ? node.allocation() : Optional.empty(), node.history().recordStateTransition(node.state(), toState, agent, clock.instant()), node.type(), node.reports(), node.modelName(), node.reservedTo(), - node.exclusiveToApplicationId(), node.hostTTL(), node.hostEmptyAt(), + node.exclusiveToApplicationId(), node.provisionedForApplicationId(), node.hostTTL(), node.hostEmptyAt(), node.exclusiveToClusterType(), node.switchHostname(), node.trustedCertificates(), node.cloudAccount(), node.wireguardPubKey()); curatorTransaction.add(createOrSet(nodePath(newNode), nodeSerializer.toJson(newNode))); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index afdedabcf71..40d8394142b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -50,7 +50,7 @@ import java.util.Optional; */ public class NodeSerializer { - // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // WARNING: Since there are multiple servers in a ZooKeeper cluster, and they upgrade one by one // (and rewrite all nodes on startup), changes to the serialized format must be made // such that what is serialized on version N+1 can be read by version N: // - ADDING FIELDS: Always ok @@ -92,6 +92,7 @@ public class NodeSerializer { private static final String modelNameKey = "modelName"; private static final String reservedToKey = "reservedTo"; private static final String exclusiveToApplicationIdKey = "exclusiveTo"; + private static final String provisionedForApplicationIdKey = "provisionedFor"; private static final String hostTTLKey = "hostTTL"; private static final String hostEmptyAtKey = "hostEmptyAt"; private static final String exclusiveToClusterTypeKey = "exclusiveToClusterType"; @@ -182,6 +183,7 @@ public class NodeSerializer { node.modelName().ifPresent(modelName -> object.setString(modelNameKey, modelName)); node.reservedTo().ifPresent(tenant -> object.setString(reservedToKey, tenant.value())); node.exclusiveToApplicationId().ifPresent(applicationId -> object.setString(exclusiveToApplicationIdKey, applicationId.serializedForm())); + node.provisionedForApplicationId().ifPresent(applicationId -> object.setString(provisionedForApplicationIdKey, applicationId.serializedForm())); node.hostTTL().ifPresent(hostTTL -> object.setLong(hostTTLKey, hostTTL.toMillis())); node.hostEmptyAt().ifPresent(emptyAt -> object.setLong(hostEmptyAtKey, emptyAt.toEpochMilli())); node.exclusiveToClusterType().ifPresent(clusterType -> object.setString(exclusiveToClusterTypeKey, clusterType.name())); @@ -281,6 +283,7 @@ public class NodeSerializer { SlimeUtils.optionalString(object.field(modelNameKey)), SlimeUtils.optionalString(object.field(reservedToKey)).map(TenantName::from), SlimeUtils.optionalString(object.field(exclusiveToApplicationIdKey)).map(ApplicationId::fromSerializedForm), + SlimeUtils.optionalString(object.field(exclusiveToApplicationIdKey)).map(ApplicationId::fromSerializedForm), // TODO: change to provisionedForApplicationIdKey SlimeUtils.optionalDuration(object.field(hostTTLKey)), SlimeUtils.optionalInstant(object.field(hostEmptyAtKey)), SlimeUtils.optionalString(object.field(exclusiveToClusterTypeKey)).map(ClusterSpec.Type::from), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index fbfa9649e59..7da80440667 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -30,7 +30,7 @@ public class ProvisionedHost { private final String hostHostname; private final Flavor hostFlavor; private final NodeType hostType; - private final Optional<ApplicationId> exclusiveToApplicationId; + private final Optional<ApplicationId> provisionedForApplicationId; private final Optional<ClusterSpec.Type> exclusiveToClusterType; private final List<HostName> nodeHostnames; private final NodeResources nodeResources; @@ -38,7 +38,7 @@ public class ProvisionedHost { private final CloudAccount cloudAccount; public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, NodeType hostType, - Optional<ApplicationId> exclusiveToApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, + Optional<ApplicationId> provisionedForApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, List<HostName> nodeHostnames, NodeResources nodeResources, Version osVersion, CloudAccount cloudAccount) { if (!hostType.isHost()) throw new IllegalArgumentException(hostType + " is not a host"); @@ -46,7 +46,7 @@ public class ProvisionedHost { this.hostHostname = Objects.requireNonNull(hostHostname, "Host hostname must be set"); this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); this.hostType = Objects.requireNonNull(hostType, "Host type must be set"); - this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId must be set"); + this.provisionedForApplicationId = Objects.requireNonNull(provisionedForApplicationId, "provisionedForApplicationId must be set"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType must be set"); this.nodeHostnames = validateNodeAddresses(nodeHostnames); this.nodeResources = Objects.requireNonNull(nodeResources, "Node resources must be set"); @@ -67,7 +67,7 @@ public class ProvisionedHost { Node.Builder builder = Node.create(id, IP.Config.of(List.of(), List.of(), nodeHostnames), hostHostname, hostFlavor, hostType) .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))) .cloudAccount(cloudAccount); - exclusiveToApplicationId.ifPresent(builder::exclusiveToApplicationId); + provisionedForApplicationId.ifPresent(builder::provisionedForApplicationId); exclusiveToClusterType.ifPresent(builder::exclusiveToClusterType); if ( ! hostTTL.isZero()) builder.hostTTL(hostTTL); return builder.build(); @@ -84,7 +84,7 @@ public class ProvisionedHost { public String hostHostname() { return hostHostname; } public Flavor hostFlavor() { return hostFlavor; } public NodeType hostType() { return hostType; } - public Optional<ApplicationId> exclusiveToApplicationId() { return exclusiveToApplicationId; } + public Optional<ApplicationId> provisionedForApplicationId() { return provisionedForApplicationId; } public Optional<ClusterSpec.Type> exclusiveToClusterType() { return exclusiveToClusterType; } public List<HostName> nodeHostnames() { return nodeHostnames; } public NodeResources nodeResources() { return nodeResources; } @@ -102,7 +102,7 @@ public class ProvisionedHost { hostHostname.equals(that.hostHostname) && hostFlavor.equals(that.hostFlavor) && hostType == that.hostType && - exclusiveToApplicationId.equals(that.exclusiveToApplicationId) && + provisionedForApplicationId.equals(that.provisionedForApplicationId) && exclusiveToClusterType.equals(that.exclusiveToClusterType) && nodeHostnames.equals(that.nodeHostnames) && nodeResources.equals(that.nodeResources) && @@ -112,7 +112,7 @@ public class ProvisionedHost { @Override public int hashCode() { - return Objects.hash(id, hostHostname, hostFlavor, hostType, exclusiveToApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); + return Objects.hash(id, hostHostname, hostFlavor, hostType, provisionedForApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); } @Override @@ -122,7 +122,7 @@ public class ProvisionedHost { ", hostHostname='" + hostHostname + '\'' + ", hostFlavor=" + hostFlavor + ", hostType=" + hostType + - ", exclusiveToApplicationId=" + exclusiveToApplicationId + + ", provisionedForApplicationId=" + provisionedForApplicationId + ", exclusiveToClusterType=" + exclusiveToClusterType + ", nodeAddresses=" + nodeHostnames + ", nodeResources=" + nodeResources + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 9825e98acdb..19b9fc26fd3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -263,6 +263,8 @@ public class NodePatcher { case "exclusiveTo": case "exclusiveToApplicationId": return node.withExclusiveToApplicationId(SlimeUtils.optionalString(value).map(ApplicationId::fromSerializedForm).orElse(null)); + case "provisionedFor": + return node.withProvisionedForApplicationId(SlimeUtils.optionalString(value).map(ApplicationId::fromSerializedForm).orElse(null)); case "hostTTL": return node.withHostTTL(SlimeUtils.optionalDuration(value).orElse(null)); case "hostEmptyAt": diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index d7cb7b4a33a..73e48d6df55 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -134,6 +134,7 @@ class NodesResponse extends SlimeJsonResponse { object.setString("flavor", node.flavor().name()); node.reservedTo().ifPresent(reservedTo -> object.setString("reservedTo", reservedTo.value())); node.exclusiveToApplicationId().ifPresent(applicationId -> object.setString("exclusiveTo", applicationId.serializedForm())); + node.provisionedForApplicationId().ifPresent(applicationId -> object.setString("provisionedFor", applicationId.serializedForm())); node.hostTTL().ifPresent(ttl -> object.setLong("hostTTL", ttl.toMillis())); node.hostEmptyAt().ifPresent(emptyAt -> object.setLong("hostEmptyAt", emptyAt.toEpochMilli())); node.exclusiveToClusterType().ifPresent(clusterType -> object.setString("exclusiveToClusterType", clusterType.name())); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 060bac4b732..1ed138625ae 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -299,6 +299,7 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { optionalString(inspector.field("parentHostname")).ifPresent(builder::parentHostname); optionalString(inspector.field("modelName")).ifPresent(builder::modelName); optionalString(inspector.field("reservedTo")).map(TenantName::from).ifPresent(builder::reservedTo); + optionalString(inspector.field("provisionedFor")).map(ApplicationId::fromSerializedForm).ifPresent(builder::provisionedForApplicationId); optionalString(inspector.field("exclusiveTo")).map(ApplicationId::fromSerializedForm).ifPresent(builder::exclusiveToApplicationId); optionalString(inspector.field("switchHostname")).ifPresent(builder::switchHostname); return builder.build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 146d2cad425..cc414cc50c2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -365,7 +365,7 @@ public class MetricsReporterTest { Capacity capacity = Capacity.from(new ClusterResources(4, 1, resources)); tester.deploy(app, spec, capacity); - // Host are now in use + // Hosts are now in use metricsReporter.maintain(); assertEquals(0, metric.values.get("nodes.emptyExclusive").intValue()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index c0958029bf5..4aab8b683b0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -480,14 +480,19 @@ public class NodeSerializerTest { assertFalse(node.hostTTL().isPresent()); assertFalse(node.exclusiveToClusterType().isPresent()); - ApplicationId exclusiveToApp = ApplicationId.from("tenant1", "app1", "instance1"); + ApplicationId provisionedForApp = ApplicationId.from("tenant1", "app1", "instance1"); + node = nodeSerializer.fromJson(nodeSerializer.toJson(builder.exclusiveToApplicationId(provisionedForApp).build())); + assertEquals(Optional.of(provisionedForApp), node.exclusiveToApplicationId()); + // assertEquals(Optional.empty(), node.provisionedForApplicationId()); TODO: enable once serialisation phase 1 is done + ClusterSpec.Type exclusiveToCluster = ClusterSpec.Type.admin; - node = builder.exclusiveToApplicationId(exclusiveToApp) + node = builder.provisionedForApplicationId(provisionedForApp) .hostTTL(Duration.ofDays(1)) .hostEmptyAt(clock.instant().minus(Duration.ofDays(1)).truncatedTo(MILLIS)) .exclusiveToClusterType(exclusiveToCluster).build(); node = nodeSerializer.fromJson(nodeSerializer.toJson(node)); - assertEquals(exclusiveToApp, node.exclusiveToApplicationId().get()); + assertEquals(provisionedForApp, node.exclusiveToApplicationId().get()); + assertEquals(provisionedForApp, node.provisionedForApplicationId().get()); assertEquals(Duration.ofDays(1), node.hostTTL().get()); assertEquals(clock.instant().minus(Duration.ofDays(1)).truncatedTo(MILLIS), node.hostEmptyAt().get()); assertEquals(exclusiveToCluster, node.exclusiveToClusterType().get()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index 973014566a0..5927cb43c3a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -238,7 +238,7 @@ public class DynamicProvisioningTest { tester.patchNodes(initialNodes.asList(), node -> node.removable(true)); NodeList exclusiveViolators = nodes.owner(application1).not().retired().first(2); List<Node> parents = exclusiveViolators.mapToList(node -> nodes.parentOf(node).get()); - tester.patchNode(parents.get(0), node -> node.withExclusiveToApplicationId(ApplicationId.defaultId())); + tester.patchNode(parents.get(0), node -> node.withProvisionedForApplicationId(ApplicationId.defaultId())); tester.patchNode(parents.get(1), node -> node.withExclusiveToClusterType(ClusterSpec.Type.container)); prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, smallerExclusiveResources, tester); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index cdbbb3b8126..72c1e2e4ec3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -1020,10 +1020,14 @@ public class NodesV2ApiTest { String url = "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"; tester.assertPartialResponse(new Request(url), "exclusiveTo", false); // Initially there is no exclusiveTo - assertResponse(new Request(url, Utf8.toBytes("{\"exclusiveToApplicationId\": \"t1:a1:i1\"}"), Request.Method.PATCH), + assertResponse(new Request(url, Utf8.toBytes("{\"exclusiveTo\": \"t1:a1:i1\"}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); tester.assertPartialResponse(new Request(url), "\"exclusiveTo\":\"t1:a1:i1\",", true); + assertResponse(new Request(url, Utf8.toBytes("{\"provisionedFor\": \"t1:a1:i1\"}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + tester.assertPartialResponse(new Request(url), "\"provisionedFor\":\"t1:a1:i1\",", true); + assertResponse(new Request(url, Utf8.toBytes("{\"hostTTL\": 86400000}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); tester.assertPartialResponse(new Request(url), "\"hostTTL\":86400000", true); @@ -1033,11 +1037,17 @@ public class NodesV2ApiTest { tester.assertPartialResponse(new Request(url), "\"hostEmptyAt\":789", true); assertResponse(new Request(url, Utf8.toBytes("{\"exclusiveToClusterType\": \"admin\"}"), Request.Method.PATCH), - "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); tester.assertPartialResponse(new Request(url), "\"exclusiveToClusterType\":\"admin\",", true); + tester.assertResponse(new Request(url, Utf8.toBytes("{\"exclusiveTo\": \"t1:a1:i2\"}"), Request.Method.PATCH), + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'exclusiveTo': exclusiveToApplicationId must be the same as provisionedForApplicationId when this is set\"}"); + + assertResponse(new Request(url, Utf8.toBytes("{\"provisionedFor\": null}"), Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); assertResponse(new Request(url, Utf8.toBytes("{\"exclusiveTo\": null, \"hostTTL\":null, \"hostEmptyAt\":null, \"exclusiveToClusterType\": null}"), Request.Method.PATCH), - "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + tester.assertPartialResponse(new Request(url), "\"exclusiveTo", false); tester.assertPartialResponse(new Request(url), "\"hostTTL\"", false); tester.assertPartialResponse(new Request(url), "\"hostEmptyAt\"", false); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json index 9979f5fc5c7..4bdd0d41999 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json @@ -7,6 +7,7 @@ "flavor": "large-variant", "reservedTo": "myTenant", "exclusiveTo": "tenant1:app1:instance1", + "provisionedFor": "tenant1:app1:instance1", "cpuCores": 64.0, "resources": { "vcpu": 64.0, diff --git a/parent/pom.xml b/parent/pom.xml index aec0f5b88fe..aed5fe071f3 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -933,7 +933,12 @@ <dependency> <groupId>org.apache.velocity</groupId> <artifactId>velocity-engine-core</artifactId> - <version>2.3</version> + <version>${velocity.vespa.version}</version> + </dependency> + <dependency> + <groupId>org.apache.velocity.tools</groupId> + <artifactId>velocity-tools-generic</artifactId> + <version>${velocity.tools.vespa.version}</version> </dependency> <dependency> <groupId>org.apiguardian</groupId> diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index e9817497904..6510808760c 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -244,6 +244,7 @@ vespa_define_module( src/tests/util/folded_string_compare src/tests/util/searchable_stats src/tests/util/slime_output_raw_buf_adapter + src/tests/util/token_extractor src/tests/vespa-fileheader-inspect ) diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/AverageAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/AverageAggregationResult.java index 5dc7cc1b634..62201db88f4 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/AverageAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/AverageAggregationResult.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class AverageAggregationResult extends AggregationResult { - public static final int classId = registerClass(0x4000 + 85, AverageAggregationResult.class); + public static final int classId = registerClass(0x4000 + 85, AverageAggregationResult.class, AverageAggregationResult::new); private NumericResultNode sum; private long count; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/CountAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/CountAggregationResult.java index 8a4fb7cdae8..3fa8db5f9db 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/CountAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/CountAggregationResult.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class CountAggregationResult extends AggregationResult { - public static final int classId = registerClass(0x4000 + 81, CountAggregationResult.class); + public static final int classId = registerClass(0x4000 + 81, CountAggregationResult.class, CountAggregationResult::new); private long count = 0; /** Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. */ diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ExpressionCountAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ExpressionCountAggregationResult.java index a02acbef281..9242e01076c 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ExpressionCountAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ExpressionCountAggregationResult.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class ExpressionCountAggregationResult extends AggregationResult { - public static final int classId = registerClass(0x4000 + 88, ExpressionCountAggregationResult.class); + public static final int classId = registerClass(0x4000 + 88, ExpressionCountAggregationResult.class, ExpressionCountAggregationResult::new); private static final int UNDEFINED = -1; // The unique count estimator @@ -29,7 +29,6 @@ public class ExpressionCountAggregationResult extends AggregationResult { /** Constructor used for deserialization. Will be instantiated with a default sketch. */ - @SuppressWarnings("UnusedDeclaration") public ExpressionCountAggregationResult() { this(new SparseSketch(), new HyperLogLogEstimator()); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java index 9b79d8d0f7b..4a29c98ad89 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class FS4Hit extends Hit { - public static final int classId = registerClass(0x4000 + 95, FS4Hit.class); // shared with c++ + public static final int classId = registerClass(0x4000 + 95, FS4Hit.class, FS4Hit::new); // shared with c++ private int path = 0; private GlobalId globalId = new GlobalId(new byte[GlobalId.LENGTH]); private int distributionKey = -1; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java index f7106d353d5..722d78a23db 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java @@ -18,7 +18,7 @@ import java.util.List; public class Group extends Identifiable { - public static final int classId = registerClass(0x4000 + 90, Group.class); + public static final int classId = registerClass(0x4000 + 90, Group.class, Group::new); private static final ObjectPredicate REF_LOCATOR = new RefLocator(); private static final int MAX_AGGREGATIONS = 0x10000; // Backend limitation private static final int MAX_ORDERBY_EXPRESSIONS = 8; // Backend limitation diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java index 2faee4ede3d..fa3e307cb4a 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java @@ -24,7 +24,7 @@ public class Grouping extends Identifiable { } // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 91, Grouping.class); + public static final int classId = registerClass(0x4000 + 91, Grouping.class, Grouping::new); // The client id for this grouping request. private int id = 0; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/GroupingLevel.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/GroupingLevel.java index ae7def70382..89f552f0bfe 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/GroupingLevel.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/GroupingLevel.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.objects.Serializer; public class GroupingLevel extends Identifiable { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 93, GroupingLevel.class); + public static final int classId = registerClass(0x4000 + 93, GroupingLevel.class, GroupingLevel::new); // The maximum number of groups allowed at this level. private long maxGroups = -1; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java index d6df04b2122..c737add21c0 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java @@ -7,8 +7,6 @@ import com.yahoo.text.Utf8; import com.yahoo.vespa.objects.*; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.List; /** @@ -20,7 +18,7 @@ import java.util.List; */ public class HitsAggregationResult extends AggregationResult { - public static final int classId = registerClass(0x4000 + 87, HitsAggregationResult.class); + public static final int classId = registerClass(0x4000 + 87, HitsAggregationResult.class, HitsAggregationResult::new); private String summaryClass = "default"; private int maxHits = -1; private List<Hit> hits = new ArrayList<>(); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/MaxAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/MaxAggregationResult.java index 0b347185f09..6d9b50b52b1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/MaxAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/MaxAggregationResult.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class MaxAggregationResult extends AggregationResult { - public static final int classId = registerClass(0x4000 + 83, MaxAggregationResult.class); + public static final int classId = registerClass(0x4000 + 83, MaxAggregationResult.class, MaxAggregationResult::new); private SingleResultNode max; /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/MinAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/MinAggregationResult.java index 0ae5587da69..1ffedb9aedc 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/MinAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/MinAggregationResult.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class MinAggregationResult extends AggregationResult { - public static final int classId = registerClass(0x4000 + 84, MinAggregationResult.class); + public static final int classId = registerClass(0x4000 + 84, MinAggregationResult.class, MinAggregationResult::new); private SingleResultNode min; /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/StandardDeviationAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/StandardDeviationAggregationResult.java index c6ece6a2525..91716f00750 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/StandardDeviationAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/StandardDeviationAggregationResult.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.objects.Serializer; * @author bjorncs */ public class StandardDeviationAggregationResult extends AggregationResult { - public static final int classId = registerClass(0x4000 + 89, StandardDeviationAggregationResult.class); + public static final int classId = registerClass(0x4000 + 89, StandardDeviationAggregationResult.class, StandardDeviationAggregationResult::new); private long count; private double sum; @@ -56,7 +56,7 @@ public class StandardDeviationAggregationResult extends AggregationResult { @Override protected boolean equalsAggregation(AggregationResult obj) { StandardDeviationAggregationResult other = (StandardDeviationAggregationResult) obj; - return count == this.count && sum == other.sum && sumOfSquared == other.sumOfSquared; + return count == other.count && sum == other.sum && sumOfSquared == other.sumOfSquared; } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/SumAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/SumAggregationResult.java index 2a78e5fde36..1f0dd0b90af 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/SumAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/SumAggregationResult.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class SumAggregationResult extends AggregationResult { - public static final int classId = registerClass(0x4000 + 82, SumAggregationResult.class); + public static final int classId = registerClass(0x4000 + 82, SumAggregationResult.class, SumAggregationResult::new); private SingleResultNode sum; /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/VdsHit.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/VdsHit.java index 6f130137008..5f14b6a937f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/VdsHit.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/VdsHit.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.objects.Serializer; public class VdsHit extends Hit { - public static final int classId = registerClass(0x4000 + 96, VdsHit.class); + public static final int classId = registerClass(0x4000 + 96, VdsHit.class, VdsHit::new); private String docId = ""; private RawData summary = new RawData(); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/XorAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/XorAggregationResult.java index fea59debd86..beb03160a05 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/XorAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/XorAggregationResult.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class XorAggregationResult extends AggregationResult { - public static final int classId = registerClass(0x4000 + 86, XorAggregationResult.class); + public static final int classId = registerClass(0x4000 + 86, XorAggregationResult.class, XorAggregationResult::new); private long xor = 0; /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/NormalSketch.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/NormalSketch.java index d7b465e230c..c27c509fa94 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/NormalSketch.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/NormalSketch.java @@ -18,7 +18,7 @@ import java.util.Arrays; */ public class NormalSketch extends Sketch<NormalSketch> { - public static final int classId = registerClass(0x4000 + 170, NormalSketch.class); + public static final int classId = registerClass(0x4000 + 170, NormalSketch.class, NormalSketch::new); private final byte[] data; private final int bucketMask; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/SparseSketch.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/SparseSketch.java index 2cff70b9e30..84896b359ef 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/SparseSketch.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/SparseSketch.java @@ -8,7 +8,7 @@ import java.util.HashSet; public class SparseSketch extends Sketch<SparseSketch> { - public static final int classId = registerClass(0x4000 + 171, SparseSketch.class); + public static final int classId = registerClass(0x4000 + 171, SparseSketch.class, SparseSketch::new); private final HashSet<Integer> values = new HashSet<>(); @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/AddFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/AddFunctionNode.java index 17524459fff..33579171650 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/AddFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/AddFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class AddFunctionNode extends NumericFunctionNode { - public static final int classId = registerClass(0x4000 + 61, AddFunctionNode.class); + public static final int classId = registerClass(0x4000 + 61, AddFunctionNode.class, AddFunctionNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/AggregationRefNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/AggregationRefNode.java index cad08090a9e..96bed811281 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/AggregationRefNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/AggregationRefNode.java @@ -14,14 +14,11 @@ import com.yahoo.vespa.objects.Serializer; */ public class AggregationRefNode extends ExpressionNode { - public static final int classId = registerClass(0x4000 + 142, AggregationRefNode.class); + public static final int classId = registerClass(0x4000 + 142, AggregationRefNode.class, AggregationRefNode::new); private AggregationResult result = null; private int index = - 1; - @SuppressWarnings("UnusedDeclaration") - public AggregationRefNode() { - // Used by deserializer. - } + public AggregationRefNode() { } public AggregationRefNode(int index) { this.index = index; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/AndFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/AndFunctionNode.java index efbcc193057..88ef0f1b7e9 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/AndFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/AndFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class AndFunctionNode extends BitFunctionNode { - public static final int classId = registerClass(0x4000 + 67, AndFunctionNode.class); + public static final int classId = registerClass(0x4000 + 67, AndFunctionNode.class, AndFunctionNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ArrayAtLookupNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ArrayAtLookupNode.java index 6fe6a57b7f9..572063eb83f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ArrayAtLookupNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ArrayAtLookupNode.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class ArrayAtLookupNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 38, ArrayAtLookupNode.class); + public static final int classId = registerClass(0x4000 + 38, ArrayAtLookupNode.class, ArrayAtLookupNode::new); private String attribute; /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/AttributeMapLookupNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/AttributeMapLookupNode.java index 580a5b0db8e..719ebcad6e1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/AttributeMapLookupNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/AttributeMapLookupNode.java @@ -17,7 +17,7 @@ import java.util.Objects; */ public class AttributeMapLookupNode extends AttributeNode { - public static final int classId = registerClass(0x4000 + 145, AttributeMapLookupNode.class); + public static final int classId = registerClass(0x4000 + 145, AttributeMapLookupNode.class, AttributeMapLookupNode::new); private String keyAttribute = ""; private String valueAttribute = ""; private String key = ""; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/AttributeNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/AttributeNode.java index 2aa6997646f..e5dac72938f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/AttributeNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/AttributeNode.java @@ -13,15 +13,13 @@ import com.yahoo.vespa.objects.Serializer; */ public class AttributeNode extends FunctionNode { - public static final int classId = registerClass(0x4000 + 55, AttributeNode.class); + public static final int classId = registerClass(0x4000 + 55, AttributeNode.class, AttributeNode::new); private String attribute; /** * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. */ - public AttributeNode() { - - } + public AttributeNode() { } /** * Constructs an instance of this class with given attribute name. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/BoolResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/BoolResultNode.java index 1a7a2a91741..b19639a62c1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/BoolResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/BoolResultNode.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.objects.Serializer; import java.nio.ByteBuffer; public class BoolResultNode extends ResultNode { - public static final int classId = registerClass(0x4000 + 146, BoolResultNode.class); + public static final int classId = registerClass(0x4000 + 146, BoolResultNode.class, BoolResultNode::new); private boolean value = false; public BoolResultNode() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/BoolResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/BoolResultNodeVector.java index fe438042ce3..62a57db17d0 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/BoolResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/BoolResultNodeVector.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.objects.Serializer; import java.util.ArrayList; public class BoolResultNodeVector extends ResultNodeVector { - public static final int classId = registerClass(0x4000 + 147, BoolResultNodeVector.class); + public static final int classId = registerClass(0x4000 + 147, BoolResultNodeVector.class, BoolResultNodeVector::new); private ArrayList<BoolResultNode> vector = new ArrayList<>(); public BoolResultNodeVector() {} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/CatFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/CatFunctionNode.java index 81c8cb3ae6b..2447603d0b2 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/CatFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/CatFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class CatFunctionNode extends MultiArgFunctionNode { - public static final int classId = registerClass(0x4000 + 72, CatFunctionNode.class); + public static final int classId = registerClass(0x4000 + 72, CatFunctionNode.class, CatFunctionNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ConstantNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ConstantNode.java index f04b4db52a9..0ef268aad69 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ConstantNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ConstantNode.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class ConstantNode extends ExpressionNode { - public static final int classId = registerClass(0x4000 + 49, ConstantNode.class); + public static final int classId = registerClass(0x4000 + 49, ConstantNode.class, ConstantNode::new); private ResultNode value = null; public ConstantNode() {} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/DebugWaitFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/DebugWaitFunctionNode.java index 97f34d2a96b..0201434ff57 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/DebugWaitFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/DebugWaitFunctionNode.java @@ -12,14 +12,11 @@ import com.yahoo.vespa.objects.Serializer; */ public class DebugWaitFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 144, DebugWaitFunctionNode.class); + public static final int classId = registerClass(0x4000 + 144, DebugWaitFunctionNode.class, DebugWaitFunctionNode::new); private double waitTime; private boolean busyWait; - @SuppressWarnings("UnusedDeclaration") - public DebugWaitFunctionNode() { - // used by deserializer - } + public DebugWaitFunctionNode() { } /** * Constructs an instance of this class with given argument and wait parameters. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/DivideFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/DivideFunctionNode.java index 07729d80053..97eded1d38f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/DivideFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/DivideFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class DivideFunctionNode extends NumericFunctionNode { - public static final int classId = registerClass(0x4000 + 63, DivideFunctionNode.class); + public static final int classId = registerClass(0x4000 + 63, DivideFunctionNode.class, DivideFunctionNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/DocumentFieldNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/DocumentFieldNode.java index 473d31bdac0..02e1ad13fc8 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/DocumentFieldNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/DocumentFieldNode.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class DocumentFieldNode extends DocumentAccessorNode { - public static final int classId = registerClass(0x4000 + 56, DocumentFieldNode.class); + public static final int classId = registerClass(0x4000 + 56, DocumentFieldNode.class, DocumentFieldNode::new); private String fieldName; private ResultNode result; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/FixedWidthBucketFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/FixedWidthBucketFunctionNode.java index cfd8fc774a6..20b9d1528ed 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/FixedWidthBucketFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/FixedWidthBucketFunctionNode.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class FixedWidthBucketFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 77, FixedWidthBucketFunctionNode.class); + public static final int classId = registerClass(0x4000 + 77, FixedWidthBucketFunctionNode.class, FixedWidthBucketFunctionNode::new); private NumericResultNode width = null; /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatBucketResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatBucketResultNode.java index b8f1431c8eb..a4191a78abd 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatBucketResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatBucketResultNode.java @@ -14,7 +14,7 @@ import com.yahoo.vespa.objects.Serializer; public class FloatBucketResultNode extends BucketResultNode { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 102, FloatBucketResultNode.class); + public static final int classId = registerClass(0x4000 + 102, FloatBucketResultNode.class, FloatBucketResultNode::new); // bucket start, inclusive private double from = 0; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatBucketResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatBucketResultNodeVector.java index 6a72b2f3787..901a0d39bbb 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatBucketResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatBucketResultNodeVector.java @@ -15,7 +15,7 @@ import java.util.ArrayList; public class FloatBucketResultNodeVector extends ResultNodeVector { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 113, FloatBucketResultNodeVector.class); + public static final int classId = registerClass(0x4000 + 113, FloatBucketResultNodeVector.class, FloatBucketResultNodeVector::new); private ArrayList<FloatBucketResultNode> vector = new ArrayList<FloatBucketResultNode>(); @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatResultNode.java index 8c1c357ab14..782e8edd661 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatResultNode.java @@ -16,9 +16,9 @@ import java.nio.ByteBuffer; public class FloatResultNode extends NumericResultNode { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 52, FloatResultNode.class); - private static FloatResultNode negativeInfinity = new FloatResultNode(Double.NEGATIVE_INFINITY); - private static FloatResultNode positiveInfinity = new FloatResultNode(Double.POSITIVE_INFINITY); + public static final int classId = registerClass(0x4000 + 52, FloatResultNode.class, FloatResultNode::new); + private static final FloatResultNode negativeInfinity = new FloatResultNode(Double.NEGATIVE_INFINITY); + private static final FloatResultNode positiveInfinity = new FloatResultNode(Double.POSITIVE_INFINITY); // The numeric value of this node. private double value; @@ -129,7 +129,7 @@ public class FloatResultNode extends NumericResultNode { @Override public Object getNumber() { - return Double.valueOf(value); + return value; } @Override @@ -141,7 +141,7 @@ public class FloatResultNode extends NumericResultNode { if (Double.isNaN(b)) { return 1; } else { - return (value < b) ? -1 : (value > b) ? 1 : 0; + return Double.compare(value, b); } } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatResultNodeVector.java index d5e69547346..b5ea963f817 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/FloatResultNodeVector.java @@ -15,7 +15,7 @@ import java.util.ArrayList; public class FloatResultNodeVector extends ResultNodeVector { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 110, FloatResultNodeVector.class); + public static final int classId = registerClass(0x4000 + 110, FloatResultNodeVector.class, FloatResultNodeVector::new); private ArrayList<FloatResultNode> vector = new ArrayList<FloatResultNode>(); @Override @@ -23,8 +23,7 @@ public class FloatResultNodeVector extends ResultNodeVector { return classId; } - public FloatResultNodeVector() { - } + public FloatResultNodeVector() {} public FloatResultNodeVector add(FloatResultNode v) { vector.add(v); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/GetDocIdNamespaceSpecificFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/GetDocIdNamespaceSpecificFunctionNode.java index 3988956d0eb..4f94d740ddf 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/GetDocIdNamespaceSpecificFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/GetDocIdNamespaceSpecificFunctionNode.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class GetDocIdNamespaceSpecificFunctionNode extends DocumentAccessorNode { - public static final int classId = registerClass(0x4000 + 73, GetDocIdNamespaceSpecificFunctionNode.class); + public static final int classId = registerClass(0x4000 + 73, GetDocIdNamespaceSpecificFunctionNode.class, GetDocIdNamespaceSpecificFunctionNode::new); private ResultNode result = null; /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java index ed110344ba1..42405e4027c 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java @@ -15,13 +15,10 @@ import java.nio.ByteBuffer; */ public class Int16ResultNode extends NumericResultNode { - public static final int classId = registerClass(0x4000 + 105, Int16ResultNode.class); + public static final int classId = registerClass(0x4000 + 105, Int16ResultNode.class, Int16ResultNode::new); private short value = 0; - @SuppressWarnings("UnusedDeclaration") - public Int16ResultNode() { - // used by deserializer - } + public Int16ResultNode() {} /** * Constructs an instance of this class with given value. @@ -122,7 +119,7 @@ public class Int16ResultNode extends NumericResultNode { @Override public Object getNumber() { - return Integer.valueOf(value); + return (int) value; } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNodeVector.java index 4e411985e66..9aea52250f9 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNodeVector.java @@ -14,11 +14,10 @@ import java.util.ArrayList; */ public class Int16ResultNodeVector extends ResultNodeVector { - public static final int classId = registerClass(0x4000 + 117, Int16ResultNodeVector.class); + public static final int classId = registerClass(0x4000 + 117, Int16ResultNodeVector.class, Int16ResultNodeVector::new); private ArrayList<Int16ResultNode> vector = new ArrayList<Int16ResultNode>(); - public Int16ResultNodeVector() { - } + public Int16ResultNodeVector() {} public Int16ResultNodeVector add(Int16ResultNode v) { vector.add(v); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java index c9f0f6c3c36..520cfb0c3d7 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java @@ -15,13 +15,10 @@ import java.nio.ByteBuffer; */ public class Int32ResultNode extends NumericResultNode { - public static final int classId = registerClass(0x4000 + 106, Int32ResultNode.class); + public static final int classId = registerClass(0x4000 + 106, Int32ResultNode.class, Int32ResultNode::new); private int value = 0; - @SuppressWarnings("UnusedDeclaration") - public Int32ResultNode() { - // used by deserializer - } + public Int32ResultNode() { } /** * Constructs an instance of this class with given value. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNodeVector.java index 8714cce52f6..6fad7f5d9ae 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNodeVector.java @@ -14,12 +14,10 @@ import java.util.ArrayList; */ public class Int32ResultNodeVector extends ResultNodeVector { - public static final int classId = registerClass(0x4000 + 118, Int32ResultNodeVector.class); + public static final int classId = registerClass(0x4000 + 118, Int32ResultNodeVector.class, Int32ResultNodeVector::new); private ArrayList<Int32ResultNode> vector = new ArrayList<Int32ResultNode>(); - public Int32ResultNodeVector() { - - } + public Int32ResultNodeVector() {} public Int32ResultNodeVector add(Int32ResultNode v) { vector.add(v); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java index 981414a473d..a2c64c931dc 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java @@ -15,11 +15,10 @@ import java.nio.ByteBuffer; */ public class Int8ResultNode extends NumericResultNode { - public static final int classId = registerClass(0x4000 + 104, Int8ResultNode.class); + public static final int classId = registerClass(0x4000 + 104, Int8ResultNode.class, Int8ResultNode::new); private byte value = 0; - public Int8ResultNode() { - } + public Int8ResultNode() { } /** * Constructs an instance of this class with given value. @@ -120,7 +119,7 @@ public class Int8ResultNode extends NumericResultNode { @Override public Object getNumber() { - return Integer.valueOf(value); + return (int) value; } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNodeVector.java index 2d00fcd2d61..a2639fa1513 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNodeVector.java @@ -14,12 +14,10 @@ import java.util.ArrayList; */ public class Int8ResultNodeVector extends ResultNodeVector { - public static final int classId = registerClass(0x4000 + 116, Int8ResultNodeVector.class); + public static final int classId = registerClass(0x4000 + 116, Int8ResultNodeVector.class, Int8ResultNodeVector::new); private ArrayList<Int8ResultNode> vector = new ArrayList<>(); - public Int8ResultNodeVector() { - - } + public Int8ResultNodeVector() {} public Int8ResultNodeVector add(Int8ResultNode v) { vector.add(v); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNode.java index a70c7b15b0c..532ae999480 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNode.java @@ -13,16 +13,11 @@ import com.yahoo.vespa.objects.Serializer; */ public class IntegerBucketResultNode extends BucketResultNode { - public static final int classId = registerClass(0x4000 + 101, IntegerBucketResultNode.class); + public static final int classId = registerClass(0x4000 + 101, IntegerBucketResultNode.class, IntegerBucketResultNode::new); private long from = 0; // bucket start, inclusive private long to = 0; // bucket end, exclusive - /** - * Constructs an empty result node. - */ - public IntegerBucketResultNode() { - // empty - } + public IntegerBucketResultNode() { } /** * Create a bucket with the given limits diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeVector.java index db33cd081c8..091da38c524 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerBucketResultNodeVector.java @@ -14,12 +14,10 @@ import java.util.ArrayList; */ public class IntegerBucketResultNodeVector extends ResultNodeVector { - public static final int classId = registerClass(0x4000 + 112, IntegerBucketResultNodeVector.class); - private ArrayList<IntegerBucketResultNode> vector = new ArrayList<IntegerBucketResultNode>(); + public static final int classId = registerClass(0x4000 + 112, IntegerBucketResultNodeVector.class, IntegerBucketResultNodeVector::new); + private ArrayList<IntegerBucketResultNode> vector = new ArrayList<>(); - public IntegerBucketResultNodeVector() { - - } + public IntegerBucketResultNodeVector() {} public IntegerBucketResultNodeVector add(IntegerBucketResultNode v) { vector.add(v); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerResultNode.java index 135c8129d96..f968a31f54c 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerResultNode.java @@ -15,9 +15,9 @@ import java.nio.ByteBuffer; */ public class IntegerResultNode extends NumericResultNode { - public static final int classId = registerClass(0x4000 + 107, IntegerResultNode.class); - private static IntegerResultNode negativeInfinity = new IntegerResultNode(Long.MIN_VALUE); - private static IntegerResultNode positiveInfinity = new IntegerResultNode(Long.MAX_VALUE); + public static final int classId = registerClass(0x4000 + 107, IntegerResultNode.class, IntegerResultNode::new); + private static final IntegerResultNode negativeInfinity = new IntegerResultNode(Long.MIN_VALUE); + private static final IntegerResultNode positiveInfinity = new IntegerResultNode(Long.MAX_VALUE); private long value; /** @@ -138,13 +138,13 @@ public class IntegerResultNode extends NumericResultNode { @Override public Object getNumber() { - return Long.valueOf(value); + return value; } @Override protected int onCmp(ResultNode rhs) { long value = rhs.getInteger(); - return (this.value < value) ? -1 : (this.value > value) ? 1 : 0; + return Long.compare(this.value, value); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerResultNodeVector.java index a840e74ede8..776213c47d0 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/IntegerResultNodeVector.java @@ -14,12 +14,10 @@ import java.util.ArrayList; */ public class IntegerResultNodeVector extends ResultNodeVector { - public static final int classId = registerClass(0x4000 + 119, IntegerResultNodeVector.class); - private ArrayList<IntegerResultNode> vector = new ArrayList<IntegerResultNode>(); + public static final int classId = registerClass(0x4000 + 119, IntegerResultNodeVector.class, IntegerResultNodeVector::new); + private ArrayList<IntegerResultNode> vector = new ArrayList<>(); - public IntegerResultNodeVector() { - - } + public IntegerResultNodeVector() {} public IntegerResultNodeVector add(IntegerResultNode v) { vector.add(v); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/InterpolatedLookupNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/InterpolatedLookupNode.java index a8175cf32d8..e8ebfad0b0d 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/InterpolatedLookupNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/InterpolatedLookupNode.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.objects.Serializer; */ public class InterpolatedLookupNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 39, InterpolatedLookupNode.class); + public static final int classId = registerClass(0x4000 + 39, InterpolatedLookupNode.class, InterpolatedLookupNode::new); private String attribute; /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/MD5BitFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/MD5BitFunctionNode.java index 6940bee45aa..81493a09e98 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/MD5BitFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/MD5BitFunctionNode.java @@ -9,14 +9,12 @@ package com.yahoo.searchlib.expression; */ public class MD5BitFunctionNode extends UnaryBitFunctionNode { - public static final int classId = registerClass(0x4000 + 70, MD5BitFunctionNode.class); + public static final int classId = registerClass(0x4000 + 70, MD5BitFunctionNode.class, MD5BitFunctionNode::new); /** * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. */ - public MD5BitFunctionNode() { - - } + public MD5BitFunctionNode() {} /** * Constructs an instance of this class with given argument and number of bits. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/MathFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/MathFunctionNode.java index e9d0b3dc069..a19ff9690ea 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/MathFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/MathFunctionNode.java @@ -53,10 +53,9 @@ public class MathFunctionNode extends MultiArgFunctionNode { } } - public static final int classId = registerClass(0x4000 + 136, MathFunctionNode.class); + public static final int classId = registerClass(0x4000 + 136, MathFunctionNode.class, MathFunctionNode::new); private Function fnc; - @SuppressWarnings("UnusedDeclaration") public MathFunctionNode() { this(Function.LOG); } @@ -75,69 +74,31 @@ public class MathFunctionNode extends MultiArgFunctionNode { @Override protected boolean onExecute() { getArg(0).execute(); - double result = 0.0; - switch (fnc) { - case EXP: - result = Math.exp(getArg(0).getResult().getFloat()); - break; - case POW: - result = Math.pow(getArg(0).getResult().getFloat(), getArg(1).getResult().getFloat()); - break; - case LOG: - result = Math.log(getArg(0).getResult().getFloat()); - break; - case LOG1P: - result = Math.log1p(getArg(0).getResult().getFloat()); - break; - case LOG10: - result = Math.log10(getArg(0).getResult().getFloat()); - break; - case SIN: - result = Math.sin(getArg(0).getResult().getFloat()); - break; - case ASIN: - result = Math.asin(getArg(0).getResult().getFloat()); - break; - case COS: - result = Math.cos(getArg(0).getResult().getFloat()); - break; - case ACOS: - result = Math.acos(getArg(0).getResult().getFloat()); - break; - case TAN: - result = Math.tan(getArg(0).getResult().getFloat()); - break; - case ATAN: - result = Math.atan(getArg(0).getResult().getFloat()); - break; - case SQRT: - result = Math.sqrt(getArg(0).getResult().getFloat()); - break; - case SINH: - result = Math.sinh(getArg(0).getResult().getFloat()); - break; - case ASINH: - throw new IllegalArgumentException("Inverse hyperbolic sine(asinh) is not supported in java"); - case COSH: - result = Math.cosh(getArg(0).getResult().getFloat()); - break; - case ACOSH: - throw new IllegalArgumentException("Inverse hyperbolic cosine (acosh) is not supported in java"); - case TANH: - result = Math.tanh(getArg(0).getResult().getFloat()); - break; - case ATANH: - throw new IllegalArgumentException("Inverse hyperbolic tangents (atanh) is not supported in java"); - case FLOOR: - result = Math.floor(getArg(0).getResult().getFloat()); - break; - case CBRT: - result = Math.cbrt(getArg(0).getResult().getFloat()); - break; - case HYPOT: - result = Math.hypot(getArg(0).getResult().getFloat(), getArg(1).getResult().getFloat()); - break; - } + double result = switch (fnc) { + case EXP -> Math.exp(getArg(0).getResult().getFloat()); + case POW -> Math.pow(getArg(0).getResult().getFloat(), getArg(1).getResult().getFloat()); + case LOG -> Math.log(getArg(0).getResult().getFloat()); + case LOG1P -> Math.log1p(getArg(0).getResult().getFloat()); + case LOG10 -> Math.log10(getArg(0).getResult().getFloat()); + case SIN -> Math.sin(getArg(0).getResult().getFloat()); + case ASIN -> Math.asin(getArg(0).getResult().getFloat()); + case COS -> Math.cos(getArg(0).getResult().getFloat()); + case ACOS -> Math.acos(getArg(0).getResult().getFloat()); + case TAN -> Math.tan(getArg(0).getResult().getFloat()); + case ATAN -> Math.atan(getArg(0).getResult().getFloat()); + case SQRT -> Math.sqrt(getArg(0).getResult().getFloat()); + case SINH -> Math.sinh(getArg(0).getResult().getFloat()); + case ASINH -> throw new IllegalArgumentException("Inverse hyperbolic sine(asinh) is not supported in java"); + case COSH -> Math.cosh(getArg(0).getResult().getFloat()); + case ACOSH -> + throw new IllegalArgumentException("Inverse hyperbolic cosine (acosh) is not supported in java"); + case TANH -> Math.tanh(getArg(0).getResult().getFloat()); + case ATANH -> + throw new IllegalArgumentException("Inverse hyperbolic tangents (atanh) is not supported in java"); + case FLOOR -> Math.floor(getArg(0).getResult().getFloat()); + case CBRT -> Math.cbrt(getArg(0).getResult().getFloat()); + case HYPOT -> Math.hypot(getArg(0).getResult().getFloat(), getArg(1).getResult().getFloat()); + }; ((FloatResultNode)getResult()).setValue(result); return true; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/MaxFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/MaxFunctionNode.java index e15d77048a6..261fbfedfea 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/MaxFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/MaxFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class MaxFunctionNode extends NumericFunctionNode { - public static final int classId = registerClass(0x4000 + 66, MaxFunctionNode.class); + public static final int classId = registerClass(0x4000 + 66, MaxFunctionNode.class, MaxFunctionNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/MinFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/MinFunctionNode.java index 71a7dbbb609..972b8181c7a 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/MinFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/MinFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class MinFunctionNode extends NumericFunctionNode { - public static final int classId = registerClass(0x4000 + 65, MinFunctionNode.class); + public static final int classId = registerClass(0x4000 + 65, MinFunctionNode.class, MinFunctionNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ModuloFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ModuloFunctionNode.java index 140ec3134f1..1d17df05e5b 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ModuloFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ModuloFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class ModuloFunctionNode extends NumericFunctionNode { - public static final int classId = registerClass(0x4000 + 64, ModuloFunctionNode.class); + public static final int classId = registerClass(0x4000 + 64, ModuloFunctionNode.class, ModuloFunctionNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/MultiArgFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/MultiArgFunctionNode.java index 49d8dd434a3..f7a34c030bd 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/MultiArgFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/MultiArgFunctionNode.java @@ -51,16 +51,16 @@ public abstract class MultiArgFunctionNode extends FunctionNode { @Override protected boolean onExecute() { - for (int i = 0; i < args.size(); i++) { - args.get(i).execute(); + for (ExpressionNode arg : args) { + arg.execute(); } return calculate(args, getResult()); } @Override protected void onPrepare() { - for (int i = 0; i < args.size(); i++) { - args.get(i).prepare(); + for (ExpressionNode arg : args) { + arg.prepare(); } prepareResult(); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/MultiplyFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/MultiplyFunctionNode.java index 2e95d7b4342..64561a7f05d 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/MultiplyFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/MultiplyFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class MultiplyFunctionNode extends NumericFunctionNode { - public static final int classId = registerClass(0x4000 + 62, MultiplyFunctionNode.class); + public static final int classId = registerClass(0x4000 + 62, MultiplyFunctionNode.class, MultiplyFunctionNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/NegateFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/NegateFunctionNode.java index a68e753f881..5be2ab2f805 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/NegateFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/NegateFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class NegateFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 60, NegateFunctionNode.class); + public static final int classId = registerClass(0x4000 + 60, NegateFunctionNode.class, NegateFunctionNode::new); /** * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/NormalizeSubjectFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/NormalizeSubjectFunctionNode.java index 7d99a1002b7..d95b8e45627 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/NormalizeSubjectFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/NormalizeSubjectFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class NormalizeSubjectFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 143, NormalizeSubjectFunctionNode.class); + public static final int classId = registerClass(0x4000 + 143, NormalizeSubjectFunctionNode.class, NormalizeSubjectFunctionNode::new); /** * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/NullResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/NullResultNode.java index 493bea276a9..a32e15e40d5 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/NullResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/NullResultNode.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.objects.ObjectVisitor; public class NullResultNode extends ResultNode { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 57, NullResultNode.class); + public static final int classId = registerClass(0x4000 + 57, NullResultNode.class, NullResultNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/NumElemFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/NumElemFunctionNode.java index 4186cf44b51..cafbde08b73 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/NumElemFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/NumElemFunctionNode.java @@ -9,14 +9,12 @@ package com.yahoo.searchlib.expression; */ public class NumElemFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 132, NumElemFunctionNode.class); + public static final int classId = registerClass(0x4000 + 132, NumElemFunctionNode.class, NumElemFunctionNode::new); /** * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. */ - public NumElemFunctionNode() { - - } + public NumElemFunctionNode() {} /** * Constructs an instance of this class with given argument. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/OrFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/OrFunctionNode.java index e507441dbbd..bdc149aabce 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/OrFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/OrFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class OrFunctionNode extends BitFunctionNode { - public static final int classId = registerClass(0x4000 + 68, OrFunctionNode.class); + public static final int classId = registerClass(0x4000 + 68, OrFunctionNode.class, OrFunctionNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/PositiveInfinityResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/PositiveInfinityResultNode.java index 1d1e5e732fa..d8872ad0e93 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/PositiveInfinityResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/PositiveInfinityResultNode.java @@ -6,7 +6,7 @@ package com.yahoo.searchlib.expression; */ public class PositiveInfinityResultNode extends ResultNode { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 124, PositiveInfinityResultNode.class); + public static final int classId = registerClass(0x4000 + 124, PositiveInfinityResultNode.class, PositiveInfinityResultNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/RangeBucketPreDefFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/RangeBucketPreDefFunctionNode.java index d6eca94cef1..42f489f69ee 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/RangeBucketPreDefFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/RangeBucketPreDefFunctionNode.java @@ -13,15 +13,10 @@ import com.yahoo.vespa.objects.Serializer; */ public class RangeBucketPreDefFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 76, RangeBucketPreDefFunctionNode.class); + public static final int classId = registerClass(0x4000 + 76, RangeBucketPreDefFunctionNode.class, RangeBucketPreDefFunctionNode::new); private ResultNodeVector predef = null; - /** - * Constructs an empty result node. - */ - public RangeBucketPreDefFunctionNode() { - // empty - } + public RangeBucketPreDefFunctionNode() {} /** * Create a bucket expression with the given width and the given subexpression diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/RawBucketResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/RawBucketResultNode.java index 6327c720d07..6d995bc1b34 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/RawBucketResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/RawBucketResultNode.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.objects.Serializer; public class RawBucketResultNode extends BucketResultNode { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 125, RawBucketResultNode.class); + public static final int classId = registerClass(0x4000 + 125, RawBucketResultNode.class, RawBucketResultNode::new); // bucket start, inclusive private ResultNode from = RawResultNode.getNegativeInfinity(); @@ -24,12 +24,7 @@ public class RawBucketResultNode extends BucketResultNode { return to.equals(from); } - /** - * Constructs an empty result node. - */ - public RawBucketResultNode() { - // empty - } + public RawBucketResultNode() {} /** * Create a bucket with the given limits diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/RawBucketResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/RawBucketResultNodeVector.java index e779eb62e17..e95caff7435 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/RawBucketResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/RawBucketResultNodeVector.java @@ -11,16 +11,15 @@ import java.util.ArrayList; */ public class RawBucketResultNodeVector extends ResultNodeVector { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 126, RawBucketResultNodeVector.class); - private ArrayList<RawBucketResultNode> vector = new ArrayList<RawBucketResultNode>(); + public static final int classId = registerClass(0x4000 + 126, RawBucketResultNodeVector.class, RawBucketResultNodeVector::new); + private ArrayList<RawBucketResultNode> vector = new ArrayList<>(); @Override protected int onGetClassId() { return classId; } - public RawBucketResultNodeVector() { - } + public RawBucketResultNodeVector() {} public RawBucketResultNodeVector add(RawBucketResultNode v) { vector.add(v); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNode.java index 78f5d529d90..3e951ae8e46 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNode.java @@ -17,7 +17,7 @@ import java.util.Arrays; public class RawResultNode extends SingleResultNode { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 54, RawResultNode.class); + public static final int classId = registerClass(0x4000 + 54, RawResultNode.class, RawResultNode::new); private static final RawResultNode negativeInfinity = new RawResultNode(); private static final PositiveInfinityResultNode positiveInfinity = new PositiveInfinityResultNode(); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNodeVector.java index fb951f4313b..8ea775799a6 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNodeVector.java @@ -15,7 +15,7 @@ import java.util.ArrayList; public class RawResultNodeVector extends ResultNodeVector { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 115, RawResultNodeVector.class); + public static final int classId = registerClass(0x4000 + 115, RawResultNodeVector.class, RawResultNodeVector::new); private ArrayList<RawResultNode> vector = new ArrayList<RawResultNode>(); @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/RelevanceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/RelevanceNode.java index 1b675d3beca..250c8902fa8 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/RelevanceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/RelevanceNode.java @@ -13,17 +13,13 @@ import com.yahoo.vespa.objects.Serializer; */ public class RelevanceNode extends ExpressionNode { - public static final int classId = registerClass(0x4000 + 59, RelevanceNode.class); + public static final int classId = registerClass(0x4000 + 59, RelevanceNode.class, RelevanceNode::new); private FloatResultNode relevance = new FloatResultNode(); - public RelevanceNode() { - - } + public RelevanceNode() {} @Override - public void onPrepare() { - - } + public void onPrepare() {} @Override public boolean onExecute() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ReverseFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ReverseFunctionNode.java index a349da71ee0..bf6d415bbf4 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ReverseFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ReverseFunctionNode.java @@ -9,14 +9,9 @@ package com.yahoo.searchlib.expression; */ public class ReverseFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 138, ReverseFunctionNode.class); + public static final int classId = registerClass(0x4000 + 138, ReverseFunctionNode.class, ReverseFunctionNode::new); - /** - * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. - */ - public ReverseFunctionNode() { - - } + public ReverseFunctionNode() {} /** * Constructs an instance of this class with given argument. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/SortFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/SortFunctionNode.java index a84cf25158e..d6780a29e74 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/SortFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/SortFunctionNode.java @@ -6,14 +6,9 @@ package com.yahoo.searchlib.expression; */ public class SortFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 137, SortFunctionNode.class); + public static final int classId = registerClass(0x4000 + 137, SortFunctionNode.class, SortFunctionNode::new); - /** - * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. - */ - public SortFunctionNode() { - - } + public SortFunctionNode() {} /** * Constructs an instance of this class with given argument. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/StrCatFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/StrCatFunctionNode.java index a1164402705..0b8a07afac3 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/StrCatFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/StrCatFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class StrCatFunctionNode extends MultiArgFunctionNode { - public static final int classId = registerClass(0x4000 + 133, StrCatFunctionNode.class); + public static final int classId = registerClass(0x4000 + 133, StrCatFunctionNode.class, StrCatFunctionNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/StrLenFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/StrLenFunctionNode.java index 406a77d2c20..bbf2b9b8097 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/StrLenFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/StrLenFunctionNode.java @@ -9,14 +9,9 @@ package com.yahoo.searchlib.expression; */ public class StrLenFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 130, StrLenFunctionNode.class); + public static final int classId = registerClass(0x4000 + 130, StrLenFunctionNode.class, StrLenFunctionNode::new); - /** - * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. - */ - public StrLenFunctionNode() { - - } + public StrLenFunctionNode() {} /** * Constructs an instance of this class with given argument. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/StringBucketResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/StringBucketResultNode.java index dd3a9d79d2f..ccd1364a581 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/StringBucketResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/StringBucketResultNode.java @@ -14,7 +14,7 @@ import com.yahoo.vespa.objects.Serializer; public class StringBucketResultNode extends BucketResultNode { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 103, StringBucketResultNode.class); + public static final int classId = registerClass(0x4000 + 103, StringBucketResultNode.class, StringBucketResultNode::new); // bucket start, inclusive private ResultNode from = StringResultNode.getNegativeInfinity(); @@ -27,12 +27,7 @@ public class StringBucketResultNode extends BucketResultNode { return to.equals(from); } - /** - * Constructs an empty result node. - */ - public StringBucketResultNode() { - // empty - } + public StringBucketResultNode() {} /** * Create a bucket with the given limits diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/StringBucketResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/StringBucketResultNodeVector.java index d3c10fec87e..41593998cb4 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/StringBucketResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/StringBucketResultNodeVector.java @@ -15,16 +15,15 @@ import java.util.ArrayList; public class StringBucketResultNodeVector extends ResultNodeVector { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 114, StringBucketResultNodeVector.class); - private ArrayList<StringBucketResultNode> vector = new ArrayList<StringBucketResultNode>(); + public static final int classId = registerClass(0x4000 + 114, StringBucketResultNodeVector.class, StringBucketResultNodeVector::new); + private ArrayList<StringBucketResultNode> vector = new ArrayList<>(); @Override protected int onGetClassId() { return classId; } - public StringBucketResultNodeVector() { - } + public StringBucketResultNodeVector() {} public StringBucketResultNodeVector add(StringBucketResultNode v) { vector.add(v); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/StringResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/StringResultNode.java index 20f204c5b61..ffd73bf2944 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/StringResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/StringResultNode.java @@ -17,7 +17,7 @@ import java.util.Arrays; public class StringResultNode extends SingleResultNode { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 53, StringResultNode.class); + public static final int classId = registerClass(0x4000 + 53, StringResultNode.class, StringResultNode::new); private static final StringResultNode negativeInfinity = new StringResultNode(""); private static final PositiveInfinityResultNode positiveInfinity = new PositiveInfinityResultNode(); @@ -92,7 +92,7 @@ public class StringResultNode extends SingleResultNode { @Override public long getInteger() { try { - return Integer.valueOf(getString()); + return Integer.parseInt(getString()); } catch (java.lang.NumberFormatException e) { return 0; } @@ -101,7 +101,7 @@ public class StringResultNode extends SingleResultNode { @Override public double getFloat() { try { - return Double.valueOf(getString()); + return Double.parseDouble(getString()); } catch (java.lang.NumberFormatException e) { return 0; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/StringResultNodeVector.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/StringResultNodeVector.java index 0c8e099e4de..142982d4f11 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/StringResultNodeVector.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/StringResultNodeVector.java @@ -15,7 +15,7 @@ import java.util.ArrayList; public class StringResultNodeVector extends ResultNodeVector { // The global class identifier shared with C++. - public static final int classId = registerClass(0x4000 + 111, StringResultNodeVector.class); + public static final int classId = registerClass(0x4000 + 111, StringResultNodeVector.class, StringResultNodeVector::new); private ArrayList<StringResultNode> vector = new ArrayList<StringResultNode>(); @Override @@ -23,8 +23,7 @@ public class StringResultNodeVector extends ResultNodeVector { return classId; } - public StringResultNodeVector() { - } + public StringResultNodeVector() {} public StringResultNodeVector add(StringResultNode v) { vector.add(v); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/TimeStampFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/TimeStampFunctionNode.java index cb5e9d1727d..6332a7134c1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/TimeStampFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/TimeStampFunctionNode.java @@ -40,14 +40,11 @@ public class TimeStampFunctionNode extends UnaryFunctionNode { } } - public static final int classId = registerClass(0x4000 + 75, TimeStampFunctionNode.class); + public static final int classId = registerClass(0x4000 + 75, TimeStampFunctionNode.class, TimeStampFunctionNode::new); private TimePart timePart = TimePart.Year; private boolean isGmt = false; - @SuppressWarnings("UnusedDeclaration") - public TimeStampFunctionNode() { - // used by deserializer - } + public TimeStampFunctionNode() {} /** * <p>Create a bucket expression with the given width and the given subexpression.</p> diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ToFloatFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ToFloatFunctionNode.java index 8553331d178..bd8941540bc 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ToFloatFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ToFloatFunctionNode.java @@ -9,14 +9,9 @@ package com.yahoo.searchlib.expression; */ public class ToFloatFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 134, ToFloatFunctionNode.class); + public static final int classId = registerClass(0x4000 + 134, ToFloatFunctionNode.class, ToFloatFunctionNode::new); - /** - * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. - */ - public ToFloatFunctionNode() { - - } + public ToFloatFunctionNode() {} /** * Constructs an instance of this class with given argument. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ToIntFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ToIntFunctionNode.java index 66b3f07cd1a..a5bf319dcda 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ToIntFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ToIntFunctionNode.java @@ -9,14 +9,9 @@ package com.yahoo.searchlib.expression; */ public class ToIntFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 135, ToIntFunctionNode.class); + public static final int classId = registerClass(0x4000 + 135, ToIntFunctionNode.class, ToIntFunctionNode::new); - /** - * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. - */ - public ToIntFunctionNode() { - - } + public ToIntFunctionNode() {} /** * Constructs an instance of this class with given argument. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ToRawFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ToRawFunctionNode.java index 9841ae04498..15480ce719c 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ToRawFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ToRawFunctionNode.java @@ -8,14 +8,9 @@ package com.yahoo.searchlib.expression; */ public class ToRawFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 141, ToRawFunctionNode.class); + public static final int classId = registerClass(0x4000 + 141, ToRawFunctionNode.class, ToRawFunctionNode::new); - /** - * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. - */ - public ToRawFunctionNode() { - - } + public ToRawFunctionNode() {} /** * Constructs an instance of this class with given argument. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ToStringFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ToStringFunctionNode.java index e42da3b6bb0..4cfc54e4341 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ToStringFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ToStringFunctionNode.java @@ -9,14 +9,9 @@ package com.yahoo.searchlib.expression; */ public class ToStringFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 131, ToStringFunctionNode.class); + public static final int classId = registerClass(0x4000 + 131, ToStringFunctionNode.class, ToStringFunctionNode::new); - /** - * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. - */ - public ToStringFunctionNode() { - - } + public ToStringFunctionNode() {} /** * Constructs an instance of this class with given argument. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/UcaFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/UcaFunctionNode.java index 4b7a589d836..df1e54b66ef 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/UcaFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/UcaFunctionNode.java @@ -12,16 +12,11 @@ import com.yahoo.vespa.objects.Serializer; */ public class UcaFunctionNode extends UnaryFunctionNode { - public static final int classId = registerClass(0x4000 + 140, UcaFunctionNode.class); + public static final int classId = registerClass(0x4000 + 140, UcaFunctionNode.class, UcaFunctionNode::new); private String locale = "en-US"; private String strength = "TERTIARY"; - /** - * Constructs an empty result node. - */ - public UcaFunctionNode() { - // empty - } + public UcaFunctionNode() {} /** * Create an UCA node with a specific locale. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/UnaryBitFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/UnaryBitFunctionNode.java index 157a0471eeb..1def3586471 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/UnaryBitFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/UnaryBitFunctionNode.java @@ -19,9 +19,7 @@ public abstract class UnaryBitFunctionNode extends UnaryFunctionNode { /** * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. */ - public UnaryBitFunctionNode() { - - } + public UnaryBitFunctionNode() {} /** * Constructs an instance of this class with given argument and number of bits. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/XorBitFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/XorBitFunctionNode.java index 313bf59b2d4..dba95e4150d 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/XorBitFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/XorBitFunctionNode.java @@ -12,14 +12,9 @@ package com.yahoo.searchlib.expression; */ public class XorBitFunctionNode extends UnaryBitFunctionNode { - public static final int classId = registerClass(0x4000 + 71, XorBitFunctionNode.class); + public static final int classId = registerClass(0x4000 + 71, XorBitFunctionNode.class, XorBitFunctionNode::new); - /** - * Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. - */ - public XorBitFunctionNode() { - - } + public XorBitFunctionNode() {} /** * Constructs an instance of this class with given argument and number of bits. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/XorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/XorFunctionNode.java index 34e10ddcc91..2cab47f8ed6 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/XorFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/XorFunctionNode.java @@ -9,7 +9,7 @@ package com.yahoo.searchlib.expression; */ public class XorFunctionNode extends BitFunctionNode { - public static final int classId = registerClass(0x4000 + 69, XorFunctionNode.class); + public static final int classId = registerClass(0x4000 + 69, XorFunctionNode.class, XorFunctionNode::new); @Override protected int onGetClassId() { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ZCurveFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ZCurveFunctionNode.java index bac3ef73c66..a068c4ed189 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ZCurveFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ZCurveFunctionNode.java @@ -32,13 +32,10 @@ public class ZCurveFunctionNode extends UnaryFunctionNode { } } - public static final int classId = registerClass(0x4000 + 139, ZCurveFunctionNode.class); + public static final int classId = registerClass(0x4000 + 139, ZCurveFunctionNode.class, ZCurveFunctionNode::new); private Dimension dim = Dimension.X; - @SuppressWarnings("UnusedDeclaration") - public ZCurveFunctionNode() { - // used by deserializer - } + private ZCurveFunctionNode() {} public ZCurveFunctionNode(ExpressionNode arg, Dimension dimension) { addArg(arg); diff --git a/searchlib/src/tests/util/token_extractor/CMakeLists.txt b/searchlib/src/tests/util/token_extractor/CMakeLists.txt new file mode 100644 index 00000000000..adfe579243c --- /dev/null +++ b/searchlib/src/tests/util/token_extractor/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_token_extractor_test_app TEST + SOURCES + token_extractor_test.cpp + DEPENDS + searchlib_test + GTest::gtest +) +vespa_add_test(NAME searchlib_token_extractor_test_app COMMAND searchlib_token_extractor_test_app) diff --git a/searchlib/src/tests/util/token_extractor/token_extractor_test.cpp b/searchlib/src/tests/util/token_extractor/token_extractor_test.cpp new file mode 100644 index 00000000000..e6944e257e9 --- /dev/null +++ b/searchlib/src/tests/util/token_extractor/token_extractor_test.cpp @@ -0,0 +1,164 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/searchlib/test/doc_builder.h> +#include <vespa/searchlib/test/string_field_builder.h> +#include <vespa/searchlib/util/token_extractor.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <variant> + +using document::DataType; +using document::Document; +using document::StringFieldValue; +using search::linguistics::TokenExtractor; +using search::test::DocBuilder; +using search::test::StringFieldBuilder; + +using AlternativeWords = std::vector<vespalib::string>; +using AlternativeWordsOrWord = std::variant<AlternativeWords, vespalib::string>; +using Words = std::vector<AlternativeWordsOrWord>; + +namespace { + +vespalib::string corrupt_word = "corruptWord"; + +vespalib::string field_name("stringfield"); + +std::unique_ptr<Document> +make_corrupted_document(DocBuilder &b, size_t wordOffset) +{ + StringFieldBuilder sfb(b); + auto doc = b.make_document("id:ns:searchdocument::18"); + doc->setValue(field_name, sfb.tokenize("before ").word(corrupt_word).tokenize(" after").build()); + vespalib::nbostream stream; + doc->serialize(stream); + std::vector<char> raw; + raw.resize(stream.size()); + stream.read(&raw[0], stream.size()); + assert(wordOffset < corrupt_word.size()); + for (size_t i = 0; i + corrupt_word.size() <= raw.size(); ++i) { + if (memcmp(&raw[i], corrupt_word.c_str(), corrupt_word.size()) == 0) { + raw[i + wordOffset] = '\0'; + break; + } + } + vespalib::nbostream badstream; + badstream.write(&raw[0], raw.size()); + return std::make_unique<Document>(b.get_repo(), badstream); +} + +} + +class TokenExtractorTest : public ::testing::Test { +protected: + using SpanTerm = TokenExtractor::SpanTerm; + DocBuilder _doc_builder; + std::unique_ptr<Document> _doc; + TokenExtractor _token_extractor; + std::vector<SpanTerm> _terms; + + static constexpr size_t max_word_len = 20; + + TokenExtractorTest(); + ~TokenExtractorTest() override; + + static DocBuilder::AddFieldsType + make_add_fields() + { + return [](auto& header) { header.addField(field_name, DataType::T_STRING); }; + } + + Words process(const StringFieldValue& value); +}; + +TokenExtractorTest::TokenExtractorTest() + : _doc_builder(make_add_fields()), + _doc(_doc_builder.make_document("id:ns:searchdocument::0")), + _token_extractor(field_name, max_word_len), + _terms() +{ +} + +TokenExtractorTest::~TokenExtractorTest() = default; + +Words +TokenExtractorTest::process(const StringFieldValue& value) +{ + Words result; + _terms.clear(); + auto span_trees = value.getSpanTrees(); + vespalib::stringref text = value.getValueRef(); + _token_extractor.extract(_terms, span_trees, text, _doc.get()); + auto it = _terms.begin(); + auto ite = _terms.end(); + auto itn = it; + for (; it != ite; ) { + for (; itn != ite && itn->span == it->span; ++itn); + if ((itn - it) > 1) { + auto& alternatives = std::get<0>(result.emplace_back()); + for (;it != itn; ++it) { + alternatives.emplace_back(it->word); + } + } else { + result.emplace_back(vespalib::string(it->word)); + ++it; + } + } + + return result; +} + +TEST_F(TokenExtractorTest, empty_string) +{ + EXPECT_EQ((Words{}), process(StringFieldValue(""))); +} + +TEST_F(TokenExtractorTest, plain_string) +{ + EXPECT_EQ((Words{"Plain string"}), process(StringFieldValue("Plain string"))); +} + +TEST_F(TokenExtractorTest, normal_string) +{ + StringFieldBuilder sfb(_doc_builder); + EXPECT_EQ((Words{"Hello", "world"}), process(sfb.tokenize("Hello world").build())); +} + +TEST_F(TokenExtractorTest, normalized_tokens) +{ + StringFieldBuilder sfb(_doc_builder); + auto value = sfb.token("Hello", false).alt_word("hello").tokenize(" world").build(); + EXPECT_EQ("Hello world", value.getValue()); + EXPECT_EQ((Words{"hello", "world"}), process(value)); +} + +TEST_F(TokenExtractorTest, alternative_tokens) +{ + StringFieldBuilder sfb(_doc_builder); + auto value = sfb.word("Hello").alt_word("hello").tokenize(" world").build(); + EXPECT_EQ("Hello world", value.getValue()); + EXPECT_EQ((Words{AlternativeWords{"Hello", "hello"}, "world"}), process(value)); +} + +TEST_F(TokenExtractorTest, word_with_nul_byte_is_truncated) +{ + auto doc = make_corrupted_document(_doc_builder, 7); + EXPECT_EQ((Words{"before", "corrupt", "after"}), process(dynamic_cast<const StringFieldValue&>(*doc->getValue(field_name)))); +} + +TEST_F(TokenExtractorTest, word_with_nul_byte_at_start_is_dropped) +{ + auto doc = make_corrupted_document(_doc_builder, 0); + EXPECT_EQ((Words{"before", "after"}), process(dynamic_cast<const StringFieldValue&>(*doc->getValue(field_name)))); +} + +TEST_F(TokenExtractorTest, too_long_word_is_dropped) +{ + StringFieldBuilder sfb(_doc_builder); + EXPECT_EQ((Words{"before", "after"}), process(sfb.tokenize("before veryverylongwordthatwillbedropped after").build())); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp index 2a54859352d..a69260c6f45 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp @@ -21,9 +21,6 @@ #include <vespa/vespalib/stllike/hash_map.hpp> #include <stdexcept> -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.memoryindex.fieldinverter"); - namespace search::memoryindex { using document::Annotation; @@ -51,45 +48,17 @@ FieldInverter::processAnnotations(const StringFieldValue &value, const Document& { _terms.clear(); auto span_trees = value.getSpanTrees(); - if (!TokenExtractor::extract(false, _terms, span_trees)) { - /* This is wrong unless field is exact match */ - const vespalib::string &text = value.getValue(); - if (text.empty()) { - return; - } - uint32_t wordRef = saveWord(text, &doc); - if (wordRef != 0u) { - add(wordRef); - stepWordPos(); - } - return; - } - const vespalib::string &text = value.getValue(); + vespalib::stringref text = value.getValueRef(); + _token_extractor.extract(_terms, span_trees, text, &doc); auto it = _terms.begin(); auto ite = _terms.end(); - uint32_t wordRef; - bool mustStep = false; for (; it != ite; ) { auto it_begin = it; - for (; it != ite && it->first == it_begin->first; ++it) { - if (it->second) { // it->second is a const FieldValue *. - wordRef = saveWord(*it->second, doc); - } else { - const Span &iSpan = it->first; - assert(iSpan.from() >= 0); - assert(iSpan.length() > 0); - wordRef = saveWord(vespalib::stringref(&text[iSpan.from()], - iSpan.length()), &doc); - } - if (wordRef != 0u) { - add(wordRef); - mustStep = true; - } - } - if (mustStep) { - stepWordPos(); - mustStep = false; + for (; it != ite && it->span == it_begin->span; ++it) { + uint32_t wordRef = saveWord(it->word); + add(wordRef); } + stepWordPos(); } } @@ -170,33 +139,19 @@ FieldInverter::endElement() } uint32_t -FieldInverter::saveWord(const vespalib::stringref word, const Document* doc) +FieldInverter::saveWord(vespalib::stringref word) { const size_t wordsSize = _words.size(); // assert((wordsSize & 3) == 0); // Check alignment - size_t len = strnlen(word.data(), word.size()); - if (len < word.size()) { - const Schema::IndexField &field = _schema.getIndexField(_fieldId); - LOG(error, "Detected NUL byte in word, length reduced from %zu to %zu, lid is %u, field is %s, truncated word is %s", word.size(), len, _docId, field.getName().c_str(), word.data()); - } - if (len > max_word_len && doc != nullptr) { - const Schema::IndexField& field = _schema.getIndexField(_fieldId); - LOG(warning, "Dropped too long word (len %zu > max len %zu) from document %s field %s, word prefix is %.100s", len, max_word_len, doc->getId().toString().c_str(), field.getName().c_str(), word.data()); - return 0u; - } - if (len == 0) { - return 0u; - } - - const size_t unpadded_size = wordsSize + 4 + len + 1; + const size_t unpadded_size = wordsSize + 4 + word.size() + 1; const size_t fullyPaddedSize = Aligner<4>::align(unpadded_size); _words.reserve(vespalib::roundUp2inN(fullyPaddedSize)); _words.resize(fullyPaddedSize); char * buf = &_words[0] + wordsSize; memset(buf, 0, 4); - memcpy(buf + 4, word.data(), len); - memset(buf + 4 + len, 0, fullyPaddedSize - unpadded_size + 1); + memcpy(buf + 4, word.data(), word.size()); + memset(buf + 4 + word.size(), 0, fullyPaddedSize - unpadded_size + 1); uint32_t wordRef = (wordsSize + 4) >> 2; // assert(wordRef != 0); @@ -204,20 +159,10 @@ FieldInverter::saveWord(const vespalib::stringref word, const Document* doc) return wordRef; } -uint32_t -FieldInverter::saveWord(const document::FieldValue &fv, const Document& doc) -{ - assert(fv.isA(FieldValue::Type::STRING)); - using RawRef = std::pair<const char*, size_t>; - RawRef sRef = fv.getAsRaw(); - return saveWord(vespalib::stringref(sRef.first, sRef.second), &doc); -} - void FieldInverter::remove(const vespalib::stringref word, uint32_t docId) { - uint32_t wordRef = saveWord(word, nullptr); - assert(wordRef != 0); + uint32_t wordRef = saveWord(word); _positions.emplace_back(wordRef, docId); } @@ -245,6 +190,17 @@ FieldInverter::endDoc() } void +FieldInverter::addWord(vespalib::stringref word, const document::Document& doc) +{ + word = _token_extractor.sanitize_word(word, &doc); + if (!word.empty()) { + uint32_t wordRef = saveWord(word); + add(wordRef); + stepWordPos(); + } +} + +void FieldInverter::processNormalDocTextField(const StringFieldValue &field, const Document& doc) { startElement(1); @@ -293,6 +249,7 @@ FieldInverter::FieldInverter(const Schema &schema, uint32_t fieldId, _docId(0), _oldPosSize(0), _schema(schema), + _token_extractor(_schema.getIndexField(_fieldId).getName(), max_word_len), _words(), _elems(), _positions(), diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 23e3f9ddfd8..4e3934ba322 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -173,6 +173,7 @@ private: uint32_t _oldPosSize; const index::Schema &_schema; + linguistics::TokenExtractor _token_extractor; WordBuffer _words; ElemInfoVec _elems; @@ -202,12 +203,7 @@ private: /** * Save the given word in the word buffer and return the word reference. */ - VESPA_DLL_LOCAL uint32_t saveWord(const vespalib::stringref word, const document::Document* doc); - - /** - * Save the field value as a word in the word buffer and return the word reference. - */ - VESPA_DLL_LOCAL uint32_t saveWord(const document::FieldValue &fv, const document::Document& doc); + VESPA_DLL_LOCAL uint32_t saveWord(vespalib::stringref word); /** * Get pointer to saved word from a word reference. @@ -326,13 +322,7 @@ public: void endDoc(); - void addWord(const vespalib::stringref word, const document::Document& doc) { - uint32_t wordRef = saveWord(word, &doc); - if (wordRef != 0u) { - add(wordRef); - stepWordPos(); - } - } + void addWord(vespalib::stringref word, const document::Document& doc); }; } diff --git a/searchlib/src/vespa/searchlib/util/token_extractor.cpp b/searchlib/src/vespa/searchlib/util/token_extractor.cpp index 555ea86d299..a78f30afe21 100644 --- a/searchlib/src/vespa/searchlib/util/token_extractor.cpp +++ b/searchlib/src/vespa/searchlib/util/token_extractor.cpp @@ -6,16 +6,25 @@ #include <vespa/document/annotation/span.h> #include <vespa/document/annotation/spanlist.h> #include <vespa/document/annotation/spantreevisitor.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/vespalib/text/utf8.h> +#include <vespa/vespalib/util/exceptions.h> + +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.util.token_extractor"); using document::AlternateSpanList; using document::Annotation; using document::AnnotationType; +using document::Document; +using document::FieldValue; using document::SimpleSpanList; using document::Span; using document::SpanList; using document::SpanNode; using document::SpanTreeVisitor; using document::StringFieldValue; +using vespalib::Utf8Reader; namespace search::linguistics { @@ -58,14 +67,85 @@ getSpan(const SpanNode &span_node) return finder.span(); } +vespalib::stringref +get_span_string_or_alternative(vespalib::stringref s, const Span &span, const FieldValue* fv) +{ + if (fv != nullptr) { + auto raw = fv->getAsRaw(); + return {raw.first, raw.second}; + } else { + return {s.data() + span.from(), static_cast<size_t>(span.length())}; + } +} + +size_t +truncated_word_len(vespalib::stringref word, size_t max_byte_len) +{ + Utf8Reader reader(word); + while (reader.hasMore()) { + auto last_pos = reader.getPos(); + (void) reader.getChar(); + if (reader.getPos() > max_byte_len) { + return last_pos; + } + } + return reader.getPos(); // No truncation +} + +constexpr size_t max_fmt_len = 100; // Max length of word in logs + +} + +TokenExtractor::TokenExtractor(const vespalib::string& field_name, size_t max_word_len) + : _field_name(field_name), + _max_word_len(max_word_len) +{ +} + +TokenExtractor::~TokenExtractor() = default; + +vespalib::stringref +TokenExtractor::sanitize_word(vespalib::stringref word, const document::Document* doc) const +{ + size_t len = strnlen(word.data(), word.size()); + if (len < word.size()) { + size_t old_len = word.size(); + len = truncated_word_len(word, len); + word = word.substr(0, len); + if (doc != nullptr) { + LOG(error, "Detected NUL byte in word, length reduced from %zu to %zu, document %s field %s, truncated word prefix is %.*s", old_len, word.size(), doc->getId().toString().c_str(), _field_name.c_str(), (int) truncated_word_len(word, max_fmt_len), word.data()); + } + } + if (word.size() > _max_word_len) { + if (doc != nullptr) { + LOG(warning, "Dropped too long word (len %zu > max len %zu) from document %s field %s, word prefix is %.*s", word.size(), _max_word_len, doc->getId().toString().c_str(), _field_name.c_str(), (int) truncated_word_len(word, max_fmt_len), word.data()); + } + return {}; + } + return word; +} + +void +TokenExtractor::consider_word(std::vector<SpanTerm>& terms, vespalib::stringref text, const Span& span, const FieldValue* fv, const Document* doc) const +{ + if (span.length() > 0 && span.from() >= 0 && + static_cast<size_t>(span.from()) + static_cast<size_t>(span.length()) <= text.size()) { + auto word = get_span_string_or_alternative(text, span, fv); + word = sanitize_word(word, doc); + if (!word.empty()) { + terms.emplace_back(span, word, fv != nullptr); + } + } } -bool -TokenExtractor::extract(bool allow_zero_length_tokens, std::vector<SpanTerm>& terms, const document::StringFieldValue::SpanTrees& trees) +void +TokenExtractor::extract(std::vector<SpanTerm>& terms, const document::StringFieldValue::SpanTrees& trees, vespalib::stringref text, const Document* doc) const { auto tree = StringFieldValue::findTree(trees, SPANTREE_NAME); if (tree == nullptr) { - return false; + /* field might not be annotated if match type is exact */ + consider_word(terms, text, Span(0, text.size()), nullptr, doc); + return; } for (const Annotation & annotation : *tree) { const SpanNode *span = annotation.getSpanNode(); @@ -73,13 +153,10 @@ TokenExtractor::extract(bool allow_zero_length_tokens, std::vector<SpanTerm>& te (annotation.getType() == *AnnotationType::TERM)) { Span sp = getSpan(*span); - if (sp.length() != 0 || allow_zero_length_tokens) { - terms.emplace_back(sp, annotation.getFieldValue()); - } + consider_word(terms, text, sp, annotation.getFieldValue(), doc); } } std::sort(terms.begin(), terms.end()); - return true; } } diff --git a/searchlib/src/vespa/searchlib/util/token_extractor.h b/searchlib/src/vespa/searchlib/util/token_extractor.h index 5796aaa7482..4955448b0c2 100644 --- a/searchlib/src/vespa/searchlib/util/token_extractor.h +++ b/searchlib/src/vespa/searchlib/util/token_extractor.h @@ -2,14 +2,16 @@ #pragma once +#include <vespa/document/annotation/span.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/vespalib/stllike/string.h> #include <vector> namespace document { -class FieldValue; -class StringFieldValue; +class Document; class Span; +class StringFieldValue; } @@ -19,9 +21,43 @@ namespace search::linguistics { * Class used to extract tokens from annotated string field value. */ class TokenExtractor { + const vespalib::string& _field_name; + size_t _max_word_len; + +public: + struct SpanTerm { + document::Span span; + vespalib::stringref word; + bool altered; + + SpanTerm(const document::Span& span_, vespalib::stringref word_, bool altered_) noexcept + : span(span_), + word(word_), + altered(altered_) + { + } + SpanTerm() noexcept + : span(), + word(), + altered(false) + { + } + bool operator<(const SpanTerm& rhs) const noexcept { + if (span != rhs.span) { + return span < rhs.span; + } + return word < rhs.word; + } + }; + +private: + void consider_word(std::vector<SpanTerm>& terms, vespalib::stringref text, const document::Span& span, const document::FieldValue* fv, const document::Document* doc) const; + public: - using SpanTerm = std::pair<document::Span, const document::FieldValue*>; - static bool extract(bool allow_zero_length_tokens, std::vector<SpanTerm>& terms, const document::StringFieldValue::SpanTrees& trees); + TokenExtractor(const vespalib::string& field_name, size_t max_word_len); + ~TokenExtractor(); + void extract(std::vector<SpanTerm>& terms, const document::StringFieldValue::SpanTrees& trees, vespalib::stringref text, const document::Document* doc) const; + vespalib::stringref sanitize_word(vespalib::stringref word, const document::Document* doc) const; }; } diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index e82ffa8d2b8..a091f8b5358 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -20,6 +20,7 @@ vespa_define_module( src/tests/docsummary/attribute_combiner src/tests/docsummary/attributedfw src/tests/docsummary/document_id_dfw + src/tests/docsummary/linguistics_tokens_converter src/tests/docsummary/matched_elements_filter src/tests/docsummary/query_term_filter_factory src/tests/docsummary/result_class diff --git a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/linguistics_tokens_converter/CMakeLists.txt new file mode 100644 index 00000000000..d9510c3a2b3 --- /dev/null +++ b/searchsummary/src/tests/docsummary/linguistics_tokens_converter/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchsummary_linguistics_tokens_converter_test_app TEST + SOURCES + linguistics_tokens_converter_test.cpp + DEPENDS + searchsummary + GTest::gtest +) + +vespa_add_test(NAME searchsummary_linguistics_tokens_converter_test_app COMMAND searchsummary_linguistics_tokens_converter_test_app) diff --git a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/linguistics_tokens_converter_test.cpp b/searchsummary/src/tests/docsummary/linguistics_tokens_converter/linguistics_tokens_converter_test.cpp new file mode 100644 index 00000000000..c8d959361ae --- /dev/null +++ b/searchsummary/src/tests/docsummary/linguistics_tokens_converter/linguistics_tokens_converter_test.cpp @@ -0,0 +1,172 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/annotation/annotation.h> +#include <vespa/document/annotation/span.h> +#include <vespa/document/annotation/spanlist.h> +#include <vespa/document/annotation/spantree.h> +#include <vespa/document/datatype/annotationtype.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/fixedtyperepo.h> +#include <vespa/searchlib/util/linguisticsannotation.h> +#include <vespa/searchsummary/docsummary/linguistics_tokens_converter.h> +#include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/data/slime/json_format.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> + +using document::Annotation; +using document::AnnotationType; +using document::DocumentType; +using document::DocumentTypeRepo; +using document::Span; +using document::SpanList; +using document::SpanTree; +using document::StringFieldValue; +using search::docsummary::LinguisticsTokensConverter; +using search::linguistics::SPANTREE_NAME; +using vespalib::SimpleBuffer; +using vespalib::Slime; +using vespalib::slime::JsonFormat; +using vespalib::slime::SlimeInserter; + +namespace { + +vespalib::string +slime_to_string(const Slime& slime) +{ + SimpleBuffer buf; + JsonFormat::encode(slime, buf, true); + return buf.get().make_string(); +} + +DocumenttypesConfig +get_document_types_config() +{ + using namespace document::config_builder; + DocumenttypesConfigBuilderHelper builder; + builder.document(42, "indexingdocument", + Struct("indexingdocument.header"), + Struct("indexingdocument.body")); + return builder.config(); +} + +} + +class LinguisticsTokensConverterTest : public testing::Test +{ +protected: + std::shared_ptr<const DocumentTypeRepo> _repo; + const DocumentType* _document_type; + document::FixedTypeRepo _fixed_repo; + + LinguisticsTokensConverterTest(); + ~LinguisticsTokensConverterTest() override; + void set_span_tree(StringFieldValue& value, std::unique_ptr<SpanTree> tree); + StringFieldValue make_annotated_string(bool alt_tokens); + StringFieldValue make_annotated_chinese_string(); + vespalib::string make_exp_annotated_chinese_string_tokens(); + vespalib::string convert(const StringFieldValue& fv); +}; + +LinguisticsTokensConverterTest::LinguisticsTokensConverterTest() + : testing::Test(), + _repo(std::make_unique<DocumentTypeRepo>(get_document_types_config())), + _document_type(_repo->getDocumentType("indexingdocument")), + _fixed_repo(*_repo, *_document_type) +{ +} + +LinguisticsTokensConverterTest::~LinguisticsTokensConverterTest() = default; + +void +LinguisticsTokensConverterTest::set_span_tree(StringFieldValue & value, std::unique_ptr<SpanTree> tree) +{ + StringFieldValue::SpanTrees trees; + trees.push_back(std::move(tree)); + value.setSpanTrees(trees, _fixed_repo); +} + +StringFieldValue +LinguisticsTokensConverterTest::make_annotated_string(bool alt_tokens) +{ + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); + tree->annotate(span_list->add(std::make_unique<Span>(0, 3)), *AnnotationType::TERM); + if (alt_tokens) { + tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), *AnnotationType::TERM); + } + tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), + Annotation(*AnnotationType::TERM, std::make_unique<StringFieldValue>("baz"))); + StringFieldValue value("foo bar"); + set_span_tree(value, std::move(tree)); + return value; +} + +StringFieldValue +LinguisticsTokensConverterTest::make_annotated_chinese_string() +{ + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); + // These chinese characters each use 3 bytes in their UTF8 encoding. + tree->annotate(span_list->add(std::make_unique<Span>(0, 15)), *AnnotationType::TERM); + tree->annotate(span_list->add(std::make_unique<Span>(15, 9)), *AnnotationType::TERM); + StringFieldValue value("我就是那个大灰狼"); + set_span_tree(value, std::move(tree)); + return value; +} + +vespalib::string +LinguisticsTokensConverterTest::make_exp_annotated_chinese_string_tokens() +{ + return R"(["我就是那个","大灰狼"])"; +} + +vespalib::string +LinguisticsTokensConverterTest::convert(const StringFieldValue& fv) +{ + LinguisticsTokensConverter converter; + Slime slime; + SlimeInserter inserter(slime); + converter.convert(fv, inserter); + return slime_to_string(slime); +} + +TEST_F(LinguisticsTokensConverterTest, convert_empty_string) +{ + vespalib::string exp(R"([])"); + StringFieldValue plain_string(""); + EXPECT_EQ(exp, convert(plain_string)); +} + +TEST_F(LinguisticsTokensConverterTest, convert_plain_string) +{ + vespalib::string exp(R"(["Foo Bar Baz"])"); + StringFieldValue plain_string("Foo Bar Baz"); + EXPECT_EQ(exp, convert(plain_string)); +} + +TEST_F(LinguisticsTokensConverterTest, convert_annotated_string) +{ + vespalib::string exp(R"(["foo","baz"])"); + auto annotated_string = make_annotated_string(false); + EXPECT_EQ(exp, convert(annotated_string)); +} + +TEST_F(LinguisticsTokensConverterTest, convert_annotated_string_with_alternatives) +{ + vespalib::string exp(R"(["foo",["bar","baz"]])"); + auto annotated_string = make_annotated_string(true); + EXPECT_EQ(exp, convert(annotated_string)); +} + +TEST_F(LinguisticsTokensConverterTest, convert_annotated_chinese_string) +{ + auto exp = make_exp_annotated_chinese_string_tokens(); + auto annotated_chinese_string = make_annotated_chinese_string(); + EXPECT_EQ(exp, convert(annotated_chinese_string)); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index 32df047c27f..e5ae47593e5 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -23,6 +23,7 @@ vespa_add_library(searchsummary_docsummary OBJECT juniper_dfw_term_visitor.cpp juniper_query_adapter.cpp juniperproperties.cpp + linguistics_tokens_converter.cpp matched_elements_filter_dfw.cpp positionsdfw.cpp query_term_filter.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp index b4f76d8e39f..bf267ab9e27 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp @@ -6,6 +6,7 @@ #include <vespa/document/annotation/span.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/juniper/juniper_separators.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> #include <vespa/searchlib/util/linguisticsannotation.h> #include <vespa/searchlib/util/token_extractor.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -17,6 +18,7 @@ using document::FieldValue; using document::Span; using document::StringFieldValue; using search::linguistics::TokenExtractor; +using search::memoryindex::FieldInverter; namespace search::docsummary { @@ -28,14 +30,7 @@ getSpanString(vespalib::stringref s, const Span &span) return {s.data() + span.from(), static_cast<size_t>(span.length())}; } -const StringFieldValue &ensureStringFieldValue(const FieldValue &value) __attribute__((noinline)); - -const StringFieldValue &ensureStringFieldValue(const FieldValue &value) { - if (!value.isA(FieldValue::Type::STRING)) { - throw vespalib::IllegalArgumentException("Illegal field type. " + value.toString(), VESPA_STRLOC); - } - return static_cast<const StringFieldValue &>(value); -} +vespalib::string dummy_field_name; } @@ -53,7 +48,7 @@ template <typename ForwardIt> void AnnotationConverter::handleAnnotations(const document::Span& span, ForwardIt it, ForwardIt last) { int annCnt = (last - it); - if (annCnt > 1 || (annCnt == 1 && it->second)) { + if (annCnt > 1 || (annCnt == 1 && it->altered)) { annotateSpans(span, it, last); } else { _out << getSpanString(_text, span) << juniper::separators::unit_separator_string; @@ -67,11 +62,7 @@ AnnotationConverter::annotateSpans(const document::Span& span, ForwardIt it, For << (getSpanString(_text, span)) << juniper::separators::interlinear_annotation_separator_string; // SEPARATOR while (it != last) { - if (it->second) { - _out << ensureStringFieldValue(*it->second).getValue(); - } else { - _out << getSpanString(_text, span); - } + _out << it->word; if (++it != last) { _out << " "; } @@ -86,26 +77,21 @@ AnnotationConverter::handleIndexingTerms(const StringFieldValue& value) using SpanTerm = TokenExtractor::SpanTerm; std::vector<SpanTerm> terms; auto span_trees = value.getSpanTrees(); - if (!TokenExtractor::extract(true, terms, span_trees)) { - // Treat a string without annotations as a single span. - SpanTerm str(Span(0, _text.size()), - static_cast<const FieldValue*>(nullptr)); - handleAnnotations(str.first, &str, &str + 1); - return; - } + TokenExtractor token_extractor(dummy_field_name, FieldInverter::max_word_len); + token_extractor.extract(terms, span_trees, _text, nullptr); auto it = terms.begin(); auto ite = terms.end(); int32_t endPos = 0; for (; it != ite; ) { auto it_begin = it; - if (it_begin->first.from() > endPos) { - Span tmpSpan(endPos, it_begin->first.from() - endPos); + if (it_begin->span.from() > endPos) { + Span tmpSpan(endPos, it_begin->span.from() - endPos); handleAnnotations(tmpSpan, it, it); - endPos = it_begin->first.from(); + endPos = it_begin->span.from(); } - for (; it != ite && it->first == it_begin->first; ++it); - handleAnnotations(it_begin->first, it_begin, it); - endPos = it_begin->first.from() + it_begin->first.length(); + for (; it != ite && it->span == it_begin->span; ++it); + handleAnnotations(it_begin->span, it_begin, it); + endPos = it_begin->span.from() + it_begin->span.length(); } int32_t wantEndPos = _text.size(); if (endPos < wantEndPos) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.cpp new file mode 100644 index 00000000000..838b0234cdb --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.cpp @@ -0,0 +1,81 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "linguistics_tokens_converter.h" +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/util/linguisticsannotation.h> +#include <vespa/searchlib/util/token_extractor.h> +#include <vespa/vespalib/data/slime/slime.h> + +using document::StringFieldValue; +using search::linguistics::TokenExtractor; +using search::memoryindex::FieldInverter; +using vespalib::Memory; +using vespalib::slime::ArrayInserter; +using vespalib::slime::Cursor; +using vespalib::slime::Inserter; + +namespace search::docsummary { + +namespace { + +vespalib::string dummy_field_name; + +} + +LinguisticsTokensConverter::LinguisticsTokensConverter() + : IStringFieldConverter(), + _text() +{ +} + +LinguisticsTokensConverter::~LinguisticsTokensConverter() = default; + +template <typename ForwardIt> +void +LinguisticsTokensConverter::handle_alternative_index_terms(ForwardIt it, ForwardIt last, Inserter& inserter) +{ + Cursor& a = inserter.insertArray(); + ArrayInserter ai(a); + for (;it != last; ++it) { + handle_index_term(it->word, ai); + } +} + +void +LinguisticsTokensConverter::handle_index_term(vespalib::stringref word, Inserter& inserter) +{ + inserter.insertString(Memory(word)); +} + +void +LinguisticsTokensConverter::handle_indexing_terms(const StringFieldValue& value, vespalib::slime::Inserter& inserter) +{ + Cursor& a = inserter.insertArray(); + ArrayInserter ai(a); + using SpanTerm = TokenExtractor::SpanTerm; + std::vector<SpanTerm> terms; + auto span_trees = value.getSpanTrees(); + TokenExtractor token_extractor(dummy_field_name, FieldInverter::max_word_len); + token_extractor.extract(terms, span_trees, _text, nullptr); + auto it = terms.begin(); + auto ite = terms.end(); + auto itn = it; + for (; it != ite; it = itn) { + for (; itn != ite && itn->span == it->span; ++itn); + if ((itn - it) > 1) { + handle_alternative_index_terms(it, itn, ai); + } else { + handle_index_term(it->word, ai); + } + } +} + +void +LinguisticsTokensConverter::convert(const StringFieldValue &input, vespalib::slime::Inserter& inserter) +{ + _text = input.getValueRef(); + handle_indexing_terms(input, inserter); +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.h new file mode 100644 index 00000000000..cba3937c822 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.h @@ -0,0 +1,28 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_string_field_converter.h" + +namespace search::docsummary { + +/* + * Class converting a string field value with annotations into an array + * containing the index terms. Multiple index terms at same position are + * placed in a nested array. + */ +class LinguisticsTokensConverter : public IStringFieldConverter +{ + vespalib::stringref _text; + + template <typename ForwardIt> + void handle_alternative_index_terms(ForwardIt it, ForwardIt last, vespalib::slime::Inserter& inserter); + void handle_index_term(vespalib::stringref word, vespalib::slime::Inserter& inserter); + void handle_indexing_terms(const document::StringFieldValue& value, vespalib::slime::Inserter& inserter); +public: + LinguisticsTokensConverter(); + ~LinguisticsTokensConverter() override; + void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; +}; + +} diff --git a/storage/src/tests/storageserver/bouncertest.cpp b/storage/src/tests/storageserver/bouncertest.cpp index c41696e1a02..5b7d279537e 100644 --- a/storage/src/tests/storageserver/bouncertest.cpp +++ b/storage/src/tests/storageserver/bouncertest.cpp @@ -51,9 +51,10 @@ struct BouncerTest : public Test { api::Timestamp timestamp, document::BucketSpace bucketSpace); - void expectMessageBouncedWithRejection(); - void expectMessageBouncedWithAbort(); - void expectMessageNotBounced(); + void expectMessageBouncedWithRejection() const; + void expect_message_bounced_with_node_down_abort() const; + void expect_message_bounced_with_shutdown_abort() const; + void expectMessageNotBounced() const; }; BouncerTest::BouncerTest() @@ -181,7 +182,7 @@ TEST_F(BouncerTest, allow_notify_bucket_change_even_when_distributor_down) { } void -BouncerTest::expectMessageBouncedWithRejection() +BouncerTest::expectMessageBouncedWithRejection() const { ASSERT_EQ(1, _upper->getNumReplies()); EXPECT_EQ(0, _upper->getNumCommands()); @@ -191,7 +192,7 @@ BouncerTest::expectMessageBouncedWithRejection() } void -BouncerTest::expectMessageBouncedWithAbort() +BouncerTest::expect_message_bounced_with_node_down_abort() const { ASSERT_EQ(1, _upper->getNumReplies()); EXPECT_EQ(0, _upper->getNumCommands()); @@ -204,7 +205,17 @@ BouncerTest::expectMessageBouncedWithAbort() } void -BouncerTest::expectMessageNotBounced() +BouncerTest::expect_message_bounced_with_shutdown_abort() const +{ + ASSERT_EQ(1, _upper->getNumReplies()); + EXPECT_EQ(0, _upper->getNumCommands()); + auto& reply = dynamic_cast<api::StorageReply&>(*_upper->getReply(0)); + EXPECT_EQ(api::ReturnCode(api::ReturnCode::ABORTED, "Node is shutting down"), reply.getResult()); + EXPECT_EQ(0, _lower->getNumCommands()); +} + +void +BouncerTest::expectMessageNotBounced() const { EXPECT_EQ(size_t(0), _upper->getNumReplies()); EXPECT_EQ(size_t(1), _lower->getNumCommands()); @@ -296,7 +307,7 @@ TEST_F(BouncerTest, abort_request_when_derived_bucket_space_node_state_is_marked auto state = makeClusterStateBundle("distributor:3 storage:3", {{ document::FixedBucketSpaces::default_space(), "distributor:3 storage:3 .2.s:d" }}); _node->getNodeStateUpdater().setClusterStateBundle(state); _upper->sendDown(createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::default_space())); - expectMessageBouncedWithAbort(); + expect_message_bounced_with_node_down_abort(); EXPECT_EQ(1, _manager->metrics().unavailable_node_aborts.getValue()); _upper->reset(); @@ -362,5 +373,23 @@ TEST_F(BouncerTest, operation_with_sufficient_bucket_bits_is_not_rejected) { expectMessageNotBounced(); } +TEST_F(BouncerTest, requests_are_rejected_after_close) { + _manager->close(); + _upper->sendDown(createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::default_space())); + expect_message_bounced_with_shutdown_abort(); +} + +TEST_F(BouncerTest, replies_are_swallowed_after_close) { + _manager->close(); + auto req = createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::default_space()); + auto reply = req->makeReply(); + _upper->sendDown(std::move(reply)); + + EXPECT_EQ(0, _upper->getNumCommands()); + EXPECT_EQ(0, _upper->getNumReplies()); + EXPECT_EQ(0, _lower->getNumCommands()); + EXPECT_EQ(0, _lower->getNumReplies()); +} + } // storage diff --git a/storage/src/vespa/storage/common/storagelink.cpp b/storage/src/vespa/storage/common/storagelink.cpp index beccd605650..ec55bc89e90 100644 --- a/storage/src/vespa/storage/common/storagelink.cpp +++ b/storage/src/vespa/storage/common/storagelink.cpp @@ -14,6 +14,23 @@ using namespace storage::api; namespace storage { +StorageLink::StorageLink(const std::string& name, + MsgDownOnFlush allow_msg_down_during_flushing, + MsgUpOnClosed allow_msg_up_during_closed) + : _name(name), + _up(nullptr), + _down(), + _state(CREATED), + _msg_down_during_flushing(allow_msg_down_during_flushing), + _msg_up_during_closed(allow_msg_up_during_closed) +{ +} + +StorageLink::StorageLink(const std::string& name) + : StorageLink(name, MsgDownOnFlush::Disallowed, MsgUpOnClosed::Disallowed) +{ +} + StorageLink::~StorageLink() { LOG(debug, "Destructing link %s.", toString().c_str()); } @@ -129,9 +146,15 @@ void StorageLink::sendDown(const StorageMessage::SP& msg) case CLOSING: case FLUSHINGDOWN: break; + case FLUSHINGUP: + if (_msg_down_during_flushing == MsgDownOnFlush::Allowed) { + break; + } + [[fallthrough]]; default: - LOG(error, "Link %s trying to send %s down while in state %s", - toString().c_str(), msg->toString().c_str(), stateToString(getState())); + LOG(error, "Link %s trying to send %s down while in state %s. Stacktrace: %s", + toString().c_str(), msg->toString().c_str(), stateToString(getState()), + vespalib::getStackTrace(0).c_str()); assert(false); } assert(msg); @@ -171,9 +194,15 @@ void StorageLink::sendUp(const std::shared_ptr<StorageMessage> & msg) case FLUSHINGDOWN: case FLUSHINGUP: break; + case CLOSED: + if (_msg_up_during_closed == MsgUpOnClosed::Allowed) { + break; + } + [[fallthrough]]; default: - LOG(error, "Link %s trying to send %s up while in state %s", - toString().c_str(), msg->toString(true).c_str(), stateToString(getState())); + LOG(error, "Link %s trying to send %s up while in state %s. Stacktrace: %s", + toString().c_str(), msg->toString(true).c_str(), stateToString(getState()), + vespalib::getStackTrace(0).c_str()); assert(false); } assert(msg); @@ -281,15 +310,14 @@ Queue::getNext(std::shared_ptr<api::StorageMessage>& msg, vespalib::duration tim void Queue::enqueue(std::shared_ptr<api::StorageMessage> msg) { - { - std::lock_guard sync(_lock); - _queue.emplace(std::move(msg)); - } + std::lock_guard sync(_lock); + _queue.emplace(std::move(msg)); _cond.notify_one(); } void Queue::signal() { + std::lock_guard sync(_lock); _cond.notify_one(); } diff --git a/storage/src/vespa/storage/common/storagelink.h b/storage/src/vespa/storage/common/storagelink.h index 2b470d029d8..3ff75df9448 100644 --- a/storage/src/vespa/storage/common/storagelink.h +++ b/storage/src/vespa/storage/common/storagelink.h @@ -41,29 +41,41 @@ public: enum State { CREATED, OPENED, CLOSING, FLUSHINGDOWN, FLUSHINGUP, CLOSED }; + enum class MsgDownOnFlush { Allowed, Disallowed }; + enum class MsgUpOnClosed { Allowed, Disallowed }; + private: - std::string _name; - StorageLink* _up; + const std::string _name; + StorageLink* _up; std::unique_ptr<StorageLink> _down; - std::atomic<State> _state; + std::atomic<State> _state; + const MsgDownOnFlush _msg_down_during_flushing; + const MsgUpOnClosed _msg_up_during_closed; public: + StorageLink(const std::string& name, + MsgDownOnFlush allow_msg_down_during_flushing, + MsgUpOnClosed allow_msg_up_during_closed); + explicit StorageLink(const std::string& name); + StorageLink(const StorageLink &) = delete; StorageLink & operator = (const StorageLink &) = delete; - StorageLink(const std::string& name) - : _name(name), _up(0), _down(), _state(CREATED) {} ~StorageLink() override; - const std::string& getName() const { return _name; } - bool isTop() const { return (_up == 0); } - bool isBottom() const { return (_down.get() == 0); } - unsigned int size() const { return (isBottom() ? 1 : _down->size() + 1); } + const std::string& getName() const noexcept { return _name; } + [[nodiscard]] bool isTop() const noexcept { return !_up; } + [[nodiscard]] bool isBottom() const noexcept { return !_down; } + [[nodiscard]] unsigned int size() const noexcept { + return (isBottom() ? 1 : _down->size() + 1); + } /** Adds the link to the end of the chain. */ void push_back(StorageLink::UP); /** Get the current state of the storage link. */ - State getState() const noexcept { return _state.load(std::memory_order_relaxed); } + [[nodiscard]] State getState() const noexcept { + return _state.load(std::memory_order_relaxed); + } /** * Called by storage server after the storage chain have been created. diff --git a/storage/src/vespa/storage/storageserver/bouncer.cpp b/storage/src/vespa/storage/storageserver/bouncer.cpp index 404058325b9..78f248e2b90 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.cpp +++ b/storage/src/vespa/storage/storageserver/bouncer.cpp @@ -22,7 +22,7 @@ LOG_SETUP(".bouncer"); namespace storage { Bouncer::Bouncer(StorageComponentRegister& compReg, const config::ConfigUri & configUri) - : StorageLink("Bouncer"), + : StorageLink("Bouncer", MsgDownOnFlush::Disallowed, MsgUpOnClosed::Allowed), _config(new vespa::config::content::core::StorBouncerConfig()), _component(compReg, "bouncer"), _lock(), @@ -30,19 +30,19 @@ Bouncer::Bouncer(StorageComponentRegister& compReg, const config::ConfigUri & co _derivedNodeStates(), _clusterState(&lib::State::UP), _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), - _metrics(std::make_unique<BouncerMetrics>()) + _metrics(std::make_unique<BouncerMetrics>()), + _closed(false) { _component.getStateUpdater().addStateListener(*this); _component.registerMetric(*_metrics); // Register for config. Normally not critical, so catching config // exception allowing program to continue if missing/faulty config. - try{ + try { if (!configUri.empty()) { _configFetcher->subscribe<vespa::config::content::core::StorBouncerConfig>(configUri.getConfigId(), this); _configFetcher->start(); } else { - LOG(info, "No config id specified. Using defaults rather than " - "config"); + LOG(info, "No config id specified. Using defaults rather than config"); } } catch (config::InvalidConfigException& e) { LOG(info, "Bouncer failed to load config '%s'. This " @@ -70,6 +70,8 @@ Bouncer::onClose() { _configFetcher->close(); _component.getStateUpdater().removeStateListener(*this); + std::lock_guard guard(_lock); + _closed = true; } void @@ -86,8 +88,7 @@ const BouncerMetrics& Bouncer::metrics() const noexcept { } void -Bouncer::validateConfig( - const vespa::config::content::core::StorBouncerConfig& newConfig) const +Bouncer::validateConfig(const vespa::config::content::core::StorBouncerConfig& newConfig) const { if (newConfig.feedRejectionPriorityThreshold != -1) { if (newConfig.feedRejectionPriorityThreshold @@ -112,12 +113,11 @@ void Bouncer::append_node_identity(std::ostream& target_stream) const { } void -Bouncer::abortCommandForUnavailableNode(api::StorageMessage& msg, - const lib::State& state) +Bouncer::abortCommandForUnavailableNode(api::StorageMessage& msg, const lib::State& state) { // If we're not up or retired, fail due to this nodes state. std::shared_ptr<api::StorageReply> reply( - static_cast<api::StorageCommand&>(msg).makeReply().release()); + static_cast<api::StorageCommand&>(msg).makeReply()); std::ostringstream ost; ost << "We don't allow command of type " << msg.getType() << " when node is in state " << state.toString(true); @@ -128,8 +128,7 @@ Bouncer::abortCommandForUnavailableNode(api::StorageMessage& msg, } void -Bouncer::rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, - int maxClockSkewInSeconds) +Bouncer::rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, int maxClockSkewInSeconds) { auto& as_cmd = dynamic_cast<api::StorageCommand&>(msg); std::ostringstream ost; @@ -140,7 +139,7 @@ Bouncer::rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, as_cmd.getSourceIndex(), ost.str().c_str()); _metrics->clock_skew_aborts.inc(); - std::shared_ptr<api::StorageReply> reply(as_cmd.makeReply().release()); + std::shared_ptr<api::StorageReply> reply(as_cmd.makeReply()); reply->setResult(api::ReturnCode(api::ReturnCode::REJECTED, ost.str())); sendUp(reply); } @@ -148,8 +147,7 @@ Bouncer::rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, void Bouncer::abortCommandDueToClusterDown(api::StorageMessage& msg, const lib::State& cluster_state) { - std::shared_ptr<api::StorageReply> reply( - static_cast<api::StorageCommand&>(msg).makeReply().release()); + std::shared_ptr<api::StorageReply> reply(static_cast<api::StorageCommand&>(msg).makeReply()); std::ostringstream ost; ost << "We don't allow external load while cluster is in state " << cluster_state.toString(true); @@ -172,35 +170,35 @@ uint64_t Bouncer::extractMutationTimestampIfAny(const api::StorageMessage& msg) { switch (msg.getType().getId()) { - case api::MessageType::PUT_ID: - return static_cast<const api::PutCommand&>(msg).getTimestamp(); - case api::MessageType::REMOVE_ID: - return static_cast<const api::RemoveCommand&>(msg).getTimestamp(); - case api::MessageType::UPDATE_ID: - return static_cast<const api::UpdateCommand&>(msg).getTimestamp(); - default: - return 0; + case api::MessageType::PUT_ID: + return static_cast<const api::PutCommand&>(msg).getTimestamp(); + case api::MessageType::REMOVE_ID: + return static_cast<const api::RemoveCommand&>(msg).getTimestamp(); + case api::MessageType::UPDATE_ID: + return static_cast<const api::UpdateCommand&>(msg).getTimestamp(); + default: + return 0; } } bool -Bouncer::isExternalLoad(const api::MessageType& type) const noexcept +Bouncer::isExternalLoad(const api::MessageType& type) noexcept { switch (type.getId()) { - case api::MessageType::PUT_ID: - case api::MessageType::REMOVE_ID: - case api::MessageType::UPDATE_ID: - case api::MessageType::GET_ID: - case api::MessageType::VISITOR_CREATE_ID: - case api::MessageType::STATBUCKET_ID: - return true; - default: - return false; + case api::MessageType::PUT_ID: + case api::MessageType::REMOVE_ID: + case api::MessageType::UPDATE_ID: + case api::MessageType::GET_ID: + case api::MessageType::VISITOR_CREATE_ID: + case api::MessageType::STATBUCKET_ID: + return true; + default: + return false; } } bool -Bouncer::isExternalWriteOperation(const api::MessageType& type) const noexcept { +Bouncer::isExternalWriteOperation(const api::MessageType& type) noexcept { switch (type.getId()) { case api::MessageType::PUT_ID: case api::MessageType::REMOVE_ID: @@ -216,8 +214,7 @@ Bouncer::rejectDueToInsufficientPriority( api::StorageMessage& msg, api::StorageMessage::Priority feedPriorityLowerBound) { - std::shared_ptr<api::StorageReply> reply( - static_cast<api::StorageCommand&>(msg).makeReply().release()); + std::shared_ptr<api::StorageReply> reply(static_cast<api::StorageCommand&>(msg).makeReply()); std::ostringstream ost; ost << "Operation priority (" << int(msg.getPriority()) << ") is lower than currently configured threshold (" @@ -231,8 +228,7 @@ Bouncer::rejectDueToInsufficientPriority( void Bouncer::reject_due_to_too_few_bucket_bits(api::StorageMessage& msg) { - std::shared_ptr<api::StorageReply> reply( - dynamic_cast<api::StorageCommand&>(msg).makeReply()); + std::shared_ptr<api::StorageReply> reply(dynamic_cast<api::StorageCommand&>(msg).makeReply()); reply->setResult(api::ReturnCode(api::ReturnCode::REJECTED, vespalib::make_string("Operation bucket %s has too few bits used (%u < minimum of %u)", msg.getBucketId().toString().c_str(), @@ -241,31 +237,22 @@ Bouncer::reject_due_to_too_few_bucket_bits(api::StorageMessage& msg) { sendUp(reply); } +void +Bouncer::reject_due_to_node_shutdown(api::StorageMessage& msg) { + std::shared_ptr<api::StorageReply> reply(dynamic_cast<api::StorageCommand&>(msg).makeReply()); + reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, "Node is shutting down")); + sendUp(reply); +} + bool Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) { - const api::MessageType& type(msg->getType()); - // All replies can come in. - if (type.isReply()) { - return false; - } - - switch (type.getId()) { - case api::MessageType::SETNODESTATE_ID: - case api::MessageType::GETNODESTATE_ID: - case api::MessageType::SETSYSTEMSTATE_ID: - case api::MessageType::ACTIVATE_CLUSTER_STATE_VERSION_ID: - case api::MessageType::NOTIFYBUCKETCHANGE_ID: - // state commands are always ok - return false; - default: - break; - } const lib::State* state; int maxClockSkewInSeconds; bool isInAvailableState; bool abortLoadWhenClusterDown; - const lib::State *cluster_state; + bool closed; + const lib::State* cluster_state; int feedPriorityLowerBound; { std::lock_guard lock(_lock); @@ -275,7 +262,34 @@ Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) cluster_state = _clusterState; isInAvailableState = state->oneOf(_config->stopAllLoadWhenNodestateNotIn.c_str()); feedPriorityLowerBound = _config->feedRejectionPriorityThreshold; + closed = _closed; + } + const api::MessageType& type = msg->getType(); + // If the node is shutting down, we want to prevent _any_ messages from reaching + // components further down the call chain. This means this case must be handled + // _before_ any logic that explicitly allows through certain message types. + if (closed) [[unlikely]] { + if (!type.isReply()) { + reject_due_to_node_shutdown(*msg); + } // else: swallow all replies + return true; } + // All replies can come in. + if (type.isReply()) { + return false; + } + switch (type.getId()) { + case api::MessageType::SETNODESTATE_ID: + case api::MessageType::GETNODESTATE_ID: + case api::MessageType::SETSYSTEMSTATE_ID: + case api::MessageType::ACTIVATE_CLUSTER_STATE_VERSION_ID: + case api::MessageType::NOTIFYBUCKETCHANGE_ID: + // state commands are always ok + return false; + default: + break; + } + // Special case for point lookup Gets while node is in maintenance mode // to allow reads to complete during two-phase cluster state transitions if ((*state == lib::State::MAINTENANCE) && (type.getId() == api::MessageType::GET_ID) && clusterIsUp(*cluster_state)) { diff --git a/storage/src/vespa/storage/storageserver/bouncer.h b/storage/src/vespa/storage/storageserver/bouncer.h index 78f07f10316..1038e94ee94 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.h +++ b/storage/src/vespa/storage/storageserver/bouncer.h @@ -41,6 +41,7 @@ class Bouncer : public StorageLink, const lib::State* _clusterState; std::unique_ptr<config::ConfigFetcher> _configFetcher; std::unique_ptr<BouncerMetrics> _metrics; + bool _closed; public: Bouncer(StorageComponentRegister& compReg, const config::ConfigUri & configUri); @@ -60,11 +61,12 @@ private: void abortCommandDueToClusterDown(api::StorageMessage&, const lib::State&); void rejectDueToInsufficientPriority(api::StorageMessage&, api::StorageMessage::Priority); void reject_due_to_too_few_bucket_bits(api::StorageMessage&); + void reject_due_to_node_shutdown(api::StorageMessage&); static bool clusterIsUp(const lib::State& cluster_state); bool isDistributor() const; - bool isExternalLoad(const api::MessageType&) const noexcept; - bool isExternalWriteOperation(const api::MessageType&) const noexcept; - bool priorityRejectionIsEnabled(int configuredPriority) const noexcept { + static bool isExternalLoad(const api::MessageType&) noexcept; + static bool isExternalWriteOperation(const api::MessageType&) noexcept; + static bool priorityRejectionIsEnabled(int configuredPriority) noexcept { return (configuredPriority != -1); } @@ -72,7 +74,7 @@ private: * If msg is a command containing a mutating timestamp (put, remove or * update commands), return that timestamp. Otherwise, return 0. */ - uint64_t extractMutationTimestampIfAny(const api::StorageMessage& msg); + static uint64_t extractMutationTimestampIfAny(const api::StorageMessage& msg); bool onDown(const std::shared_ptr<api::StorageMessage>&) override; void handleNewState() noexcept override; const lib::NodeState &getDerivedNodeState(document::BucketSpace bucketSpace) const; diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 610d9c8d707..5bbc5b2a26d 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -217,7 +217,7 @@ convert_to_rpc_compression_config(const vespa::config::content::core::StorCommun } CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, const config::ConfigUri & configUri) - : StorageLink("Communication manager"), + : StorageLink("Communication manager", MsgDownOnFlush::Allowed, MsgUpOnClosed::Disallowed), _component(compReg, "communicationmanager"), _metrics(), _shared_rpc_resources(), // Created upon initial configuration @@ -278,25 +278,25 @@ CommunicationManager::onClose() // Avoid getting config during shutdown _configFetcher.reset(); - _closed = true; - - if (_mbus) { - if (_messageBusSession) { - _messageBusSession->close(); - } - } - - // TODO remove? this no longer has any particularly useful semantics + _closed.store(true, std::memory_order_seq_cst); if (_cc_rpc_service) { - _cc_rpc_service->close(); + _cc_rpc_service->close(); // Auto-abort all incoming CC RPC requests from now on } - // TODO do this after we drain queues? + // Sync all RPC threads to ensure that any subsequent RPCs must observe the closed-flags we just set if (_shared_rpc_resources) { - _shared_rpc_resources->shutdown(); + _shared_rpc_resources->sync_all_threads(); + } + + if (_mbus && _messageBusSession) { + // Closing the mbus session unregisters the destination session and syncs the worker + // thread(s), so once this call returns we should not observe further incoming requests + // through this pipeline. Previous messages may already be in flight internally; these + // will be handled by flushing-phases. + _messageBusSession->close(); } - // Stopping pumper thread should stop all incoming messages from being - // processed. + // Stopping internal message dispatch thread should stop all incoming _async_ messages + // from being processed. _Synchronously_ dispatched RPCs are still passing through. if (_thread) { _thread->interrupt(); _eventQueue.signal(); @@ -305,13 +305,12 @@ CommunicationManager::onClose() } // Emptying remaining queued messages - // FIXME but RPC/mbus is already shut down at this point...! Make sure we handle this std::shared_ptr<api::StorageMessage> msg; api::ReturnCode code(api::ReturnCode::ABORTED, "Node shutting down"); while (_eventQueue.size() > 0) { assert(_eventQueue.getNext(msg, 0ms)); if (!msg->getType().isReply()) { - std::shared_ptr<api::StorageReply> reply(static_cast<api::StorageCommand&>(*msg).makeReply()); + std::shared_ptr<api::StorageReply> reply(dynamic_cast<api::StorageCommand&>(*msg).makeReply()); reply->setResult(code); sendReply(reply); } @@ -319,6 +318,29 @@ CommunicationManager::onClose() } void +CommunicationManager::onFlush(bool downwards) +{ + if (downwards) { + // Sync RPC threads once more (with feeling!) to ensure that any closing done by other components + // during the storage chain onClose() is visible to these. + if (_shared_rpc_resources) { + _shared_rpc_resources->sync_all_threads(); + } + // By this point, no inbound RPCs (requests and responses) should be allowed any further down + // than the Bouncer component, where they will be, well, bounced. + } else { + // All components further down the storage chain should now be completely closed + // and flushed, and all message-dispatching threads should have been shut down. + // It's possible that the RPC threads are still butting heads up against the Bouncer + // component, so we conclude the shutdown ceremony by taking down the RPC subsystem. + // This transitively waits for all RPC threads to complete. + if (_shared_rpc_resources) { + _shared_rpc_resources->shutdown(); + } + } +} + +void CommunicationManager::configureMessageBusLimits(const CommunicationManagerConfig& cfg) { const bool isDist(_component.getNodeType() == lib::NodeType::DISTRIBUTOR); @@ -438,11 +460,15 @@ CommunicationManager::process(const std::shared_ptr<api::StorageMessage>& msg) } } +// Called directly by RPC threads void CommunicationManager::dispatch_sync(std::shared_ptr<api::StorageMessage> msg) { LOG(spam, "Direct dispatch of storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); + // If process is shutting down, msg will be synchronously aborted by the Bouncer component process(msg); } +// Called directly by RPC threads (for incoming CC requests) and by any other request-dispatching +// threads (i.e. calling sendUp) when address resolution fails and an internal error response is generated. void CommunicationManager::dispatch_async(std::shared_ptr<api::StorageMessage> msg) { LOG(spam, "Enqueued dispatch of storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); _eventQueue.enqueue(std::move(msg)); diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index da45124ed2d..3c986c59c5e 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -89,6 +89,7 @@ private: void onOpen() override; void onClose() override; + void onFlush(bool downwards) override; void process(const std::shared_ptr<api::StorageMessage>& msg); diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp index 172084662e2..eb933f5eb2c 100644 --- a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp @@ -105,6 +105,10 @@ void SharedRpcResources::wait_until_slobrok_is_ready() { } } +void SharedRpcResources::sync_all_threads() { + _transport->sync(); +} + void SharedRpcResources::shutdown() { assert(!_shutdown); if (listen_port() > 0) { diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h index 1da89dd8869..d8f7eefad53 100644 --- a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h @@ -42,6 +42,8 @@ public: // To be called after all RPC handlers have been registered. void start_server_and_register_slobrok(vespalib::stringref my_handle); + void sync_all_threads(); + void shutdown(); [[nodiscard]] int listen_port() const noexcept; // Only valid if server has been started diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index 3231deef268..452a94496af 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -358,7 +358,7 @@ StorageNode::shutdown() { // Try to shut down in opposite order of initialize. Bear in mind that // we might be shutting down after init exception causing only parts - // of the server to have initialize + // of the server to have been initialized LOG(debug, "Shutting down storage node of type %s", getNodeType().toString().c_str()); if (!attemptedStopped()) { LOG(debug, "Storage killed before requestShutdown() was called. No " diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 619b032d24f..7f6e29f6957 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -20,6 +20,7 @@ com.fasterxml.jackson.core:jackson-databind:${jackson-databind.vespa.version} com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson2.vespa.version} com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jackson2.vespa.version} com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jackson2.vespa.version} +com.github.cliftonlabs:json-simple:${findbugs.vespa.version} com.github.luben:zstd-jni:${luben.zstd.vespa.version} com.github.spotbugs:spotbugs-annotations:3.1.9 com.google.code.findbugs:jsr305:${findbugs.vespa.version} @@ -43,8 +44,10 @@ com.yahoo.athenz:athenz-zms-core:${athenz.vespa.version} com.yahoo.athenz:athenz-zpe-java-client:${athenz.vespa.version} com.yahoo.athenz:athenz-zts-core:${athenz.vespa.version} com.yahoo.rdl:rdl-java:1.5.4 +commons-beanutils:commons-beanutils:${commons-beanutils.vespa.version} commons-cli:commons-cli:1.5.0 commons-codec:commons-codec:${commons-codec.vespa.version} +commons-collections:commons-collections:${commons-collections.vespa.version} commons-fileupload:commons-fileupload:1.5 commons-io:commons-io:${commons-io.vespa.version} commons-logging:commons-logging:${commons-logging.vespa.version} @@ -87,6 +90,7 @@ org.antlr:antlr4-runtime:${antlr4.vespa.version} org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:${spifly.vespa.version} org.apache.commons:commons-compress:${commons-compress.vespa.version} org.apache.commons:commons-csv:${commons-csv.vespa.version} +org.apache.commons:commons-digester3:${commons-digester.vespa.version} org.apache.commons:commons-exec:${commons-exec.vespa.version} org.apache.commons:commons-lang3:${commons-lang3.vespa.version} org.apache.commons:commons-math3:${commons.math3.vespa.version} @@ -119,7 +123,8 @@ org.apache.maven:maven-project:2.2.1 org.apache.maven:maven-repository-metadata:${maven-core.vespa.version} org.apache.maven:maven-settings:${maven-core.vespa.version} org.apache.opennlp:opennlp-tools:${opennlp.vespa.version} -org.apache.velocity:velocity-engine-core:2.3 +org.apache.velocity.tools:velocity-tools-generic:${velocity.tools.vespa.version} +org.apache.velocity:velocity-engine-core:${velocity.vespa.version} org.apache.yetus:audience-annotations:0.12.0 org.apache.zookeeper:zookeeper-jute:${zookeeper.client.vespa.version} org.apache.zookeeper:zookeeper-jute:3.8.1 diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 43e74af9c07..5cd1d3bf9b6 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -851,8 +851,12 @@ public class DocumentGenMojo extends AbstractMojo { for (Field field: fields) { DataType dt = field.getDataType(); out.write( - ind(ind)+"public "+toJavaType(dt)+" "+getter(field.getName())+"() { return "+field.getName()+"; }\n"+ - ind(ind)+"public "+className+" "+setter(field.getName())+"("+toJavaType(dt)+" "+field.getName()+") { this."+field.getName()+"="+field.getName()+"; return this; }\n"); + ind(ind) + "public " + toJavaType(dt) + " " + getter(field.getName()) + "() { return " + field.getName() + "; }\n" + + ind(ind) + "public " + className + " " + setter(field.getName()) + "(" + toJavaType(dt) + " " + field.getName() + ") {\n" + + validateArgument(field.getDataType(), field.getName(), ind + 1) + + ind(ind+1) + "this." + field.getName() + "=" + field.getName() + ";\n" + + ind(ind+1) + "return this;\n" + + ind(ind) + "}\n"); if (spanTrees && dt.equals(DataType.STRING)) { out.write(ind(ind)+"public java.util.Map<java.lang.String,com.yahoo.document.annotation.SpanTree> "+spanTreeGetter(field.getName())+"() { return "+field.getName()+"SpanTrees; }\n" + ind(ind)+"public void "+spanTreeSetter(field.getName())+"(java.util.Map<java.lang.String,com.yahoo.document.annotation.SpanTree> spanTrees) { this."+field.getName()+"SpanTrees=spanTrees; }\n"); @@ -861,6 +865,38 @@ public class DocumentGenMojo extends AbstractMojo { out.write("\n"); } + private static String validateArgument(DataType type, String variable, int ind) { + if (type instanceof MapDataType mdt) { + return validateWrapped(mdt.getKeyType(), variable, variable + ".keySet()", ind) + + validateWrapped(mdt.getValueType(), variable, variable + ".values()", ind); + } + else if (type instanceof CollectionDataType cdt) { + String elements = cdt instanceof WeightedSetDataType ? variable + ".keySet()" : variable; + return validateWrapped(cdt.getNestedType(), variable, elements, ind); + } + else if ( DataType.STRING.equals(type) + || DataType.URI.equals(type) + || type instanceof AnnotationReferenceDataType + || type instanceof NewDocumentReferenceDataType) { + return ind(ind) + "if (" + variable + " != null) {\n" + + ind(ind+1) + toJavaReference(type) + ".createFieldValue(" + variable + ");\n" + + ind(ind) + "}\n"; + } + else { + return ""; + } + } + + private static String validateWrapped(DataType type, String variable, String elements, int ind) { + String wrappedValidation = validateArgument(type, variable + "$", ind + 2); + if (wrappedValidation.isBlank()) return ""; + return ind(ind) + "if (" + variable + " != null) {\n" + + ind(ind+1) + "for (" + toJavaType(type) + " " + variable + "$ : " + elements + ") {\n" + + wrappedValidation + + ind(ind+1) + "}\n" + + ind(ind) + "}\n"; + } + private static String spanTreeSetter(String field) { return setter(field)+"SpanTrees"; } @@ -914,10 +950,10 @@ public class DocumentGenMojo extends AbstractMojo { if (DataType.BOOL.equals(dt)) return "java.lang.Boolean"; if (DataType.TAG.equals(dt)) return "java.lang.String"; if (dt instanceof StructDataType) return className(dt.getName()); - if (dt instanceof WeightedSetDataType) return "java.util.Map<"+toJavaType(((WeightedSetDataType)dt).getNestedType())+",java.lang.Integer>"; - if (dt instanceof ArrayDataType) return "java.util.List<"+toJavaType(((ArrayDataType)dt).getNestedType())+">"; - if (dt instanceof MapDataType) return "java.util.Map<"+toJavaType(((MapDataType)dt).getKeyType())+","+toJavaType(((MapDataType)dt).getValueType())+">"; - if (dt instanceof AnnotationReferenceDataType) return className(((AnnotationReferenceDataType) dt).getAnnotationType().getName()); + if (dt instanceof WeightedSetDataType wdt) return "java.util.Map<"+toJavaType(wdt.getNestedType())+",java.lang.Integer>"; + if (dt instanceof ArrayDataType adt) return "java.util.List<"+toJavaType(adt.getNestedType())+">"; + if (dt instanceof MapDataType mdt) return "java.util.Map<"+toJavaType(mdt.getKeyType())+","+toJavaType((mdt).getValueType())+">"; + if (dt instanceof AnnotationReferenceDataType ardt) return className(ardt.getAnnotationType().getName()); if (dt instanceof NewDocumentReferenceDataType) { return "com.yahoo.document.DocumentId"; } @@ -942,22 +978,22 @@ public class DocumentGenMojo extends AbstractMojo { if (DataType.BOOL.equals(dt)) return "com.yahoo.document.DataType.BOOL"; if (DataType.TAG.equals(dt)) return "com.yahoo.document.DataType.TAG"; if (dt instanceof StructDataType) return className(dt.getName()) +".type"; - if (dt instanceof WeightedSetDataType) return "new com.yahoo.document.WeightedSetDataType("+toJavaReference(((WeightedSetDataType)dt).getNestedType())+", "+ - ((WeightedSetDataType)dt).createIfNonExistent()+", "+ ((WeightedSetDataType)dt).removeIfZero()+","+dt.getId()+")"; - if (dt instanceof ArrayDataType) return "new com.yahoo.document.ArrayDataType("+toJavaReference(((ArrayDataType)dt).getNestedType())+")"; - if (dt instanceof MapDataType) return "new com.yahoo.document.MapDataType("+toJavaReference(((MapDataType)dt).getKeyType())+", "+ - toJavaReference(((MapDataType)dt).getValueType())+", "+dt.getId()+")"; + if (dt instanceof WeightedSetDataType wdt) return "new com.yahoo.document.WeightedSetDataType("+toJavaReference(wdt.getNestedType())+", "+ + wdt.createIfNonExistent()+", "+ wdt.removeIfZero()+","+dt.getId()+")"; + if (dt instanceof ArrayDataType adt) return "new com.yahoo.document.ArrayDataType("+toJavaReference(adt.getNestedType())+")"; + if (dt instanceof MapDataType mdt) return "new com.yahoo.document.MapDataType("+toJavaReference(mdt.getKeyType())+", "+ + toJavaReference(mdt.getValueType())+", "+dt.getId()+")"; // For annotation references and generated types, the references are to the actual objects of the correct types, so most likely this is never needed, // but there might be scenarios where we want to look up the AnnotationType in the AnnotationTypeRegistry here instead. - if (dt instanceof AnnotationReferenceDataType) return "new com.yahoo.document.annotation.AnnotationReferenceDataType(new com.yahoo.document.annotation.AnnotationType(\""+((AnnotationReferenceDataType)dt).getAnnotationType().getName()+"\"))"; - if (dt instanceof NewDocumentReferenceDataType) { + if (dt instanceof AnnotationReferenceDataType adt) return "new com.yahoo.document.annotation.AnnotationReferenceDataType(new com.yahoo.document.annotation.AnnotationType(\""+adt.getAnnotationType().getName()+"\"))"; + if (dt instanceof NewDocumentReferenceDataType nrdt) { // All concrete document types have a public `type` constant with their DocumentType. return String.format("new com.yahoo.document.ReferenceDataType(%s.type, %d)", - className(((NewDocumentReferenceDataType) dt).getTargetType().getName()), dt.getId()); + className(nrdt.getTargetType().getName()), dt.getId()); } - if (dt instanceof TensorDataType) { + if (dt instanceof TensorDataType tdt) { return String.format("new com.yahoo.document.TensorDataType(com.yahoo.tensor.TensorType.fromSpec(\"%s\"))", - ((TensorDataType)dt).getTensorType().toString()); + tdt.getTensorType().toString()); } return "com.yahoo.document.DataType.RAW"; } diff --git a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java index a2820e12309..d0315f84272 100644 --- a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java +++ b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java @@ -11,8 +11,9 @@ import java.io.File; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; public class DocumentGenTest { @@ -39,7 +40,7 @@ public class DocumentGenTest { assertEquals(searches.get("music3").getDocument("music3").getField("pos").getDataType().getName(), "position"); assertTrue(searches.get("book").getDocument("book").getField("mystruct").getDataType() instanceof StructDataType); assertTrue(searches.get("book").getDocument("book").getField("mywsinteger").getDataType() instanceof WeightedSetDataType); - assertTrue(((WeightedSetDataType)(searches.get("book").getDocument("book").getField("mywsinteger").getDataType())).getNestedType() == DataType.INT); + assertSame(((WeightedSetDataType) (searches.get("book").getDocument("book").getField("mywsinteger").getDataType())).getNestedType(), DataType.INT); } @Test @@ -50,18 +51,15 @@ public class DocumentGenTest { assertEquals(searches.get("video").getDocument("video").getField("weight").getDataType(), DataType.FLOAT); assertTrue(searches.get("book").getDocument("book").getField("mystruct").getDataType() instanceof StructDataType); assertTrue(searches.get("book").getDocument("book").getField("mywsinteger").getDataType() instanceof WeightedSetDataType); - assertTrue(((WeightedSetDataType)(searches.get("book").getDocument("book").getField("mywsinteger").getDataType())).getNestedType() == DataType.INT); + assertSame(((WeightedSetDataType) (searches.get("book").getDocument("book").getField("mywsinteger").getDataType())).getNestedType(), DataType.INT); } @Test public void testEmptyPkgNameForbidden() { - DocumentGenMojo mojo = new DocumentGenMojo(); - try { - mojo.execute(new File("etc/localapp/"), new File("target/generated-test-sources/vespa-documentgen-plugin/"), ""); - fail("Didn't throw in empty pkg"); - } catch (IllegalArgumentException e) { - - } + assertThrows(IllegalArgumentException.class, + () -> new DocumentGenMojo().execute(new File("etc/localapp/"), + new File("target/generated-test-sources/vespa-documentgen-plugin/"), + "")); } } diff --git a/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesTest.java b/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesTest.java index d270284fe31..e516ae06d8f 100644 --- a/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesTest.java +++ b/vespa-enforcer-extensions/src/test/java/com/yahoo/vespa/maven/plugin/enforcer/EnforceDependenciesTest.java @@ -1,5 +1,5 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.maven.plugin.enforcer;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.maven.plugin.enforcer; import org.apache.maven.artifact.Artifact; import org.apache.maven.artifact.DefaultArtifact; diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index a0d739ef750..1c19c2ba5d6 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -3567,6 +3567,7 @@ "public boolean equals(java.lang.Object)", "public java.lang.String toString()", "protected static int registerClass(int, java.lang.Class)", + "protected static int registerClass(int, java.lang.Class, java.util.function.Supplier)", "public static com.yahoo.vespa.objects.Identifiable create(com.yahoo.vespa.objects.Deserializer)", "public static com.yahoo.vespa.objects.Identifiable createFromId(int)", "protected static com.yahoo.vespa.objects.Serializer serializeOptional(com.yahoo.vespa.objects.Serializer, com.yahoo.vespa.objects.Identifiable)", diff --git a/vespajlib/src/main/java/com/yahoo/slime/Cursor.java b/vespajlib/src/main/java/com/yahoo/slime/Cursor.java index a80e5e65cce..60eeb1e2218 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Cursor.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Cursor.java @@ -94,7 +94,7 @@ public interface Cursor extends Inspector { /** * Appends an array entry containing a new value of ARRAY type. - * Returns a valid Cursor (thay may again be used for adding new + * Returns a valid Cursor (that may again be used for adding new * sub-array entries) referencing the new entry value if * successful; otherwise returns an invalid Cursor. * @@ -104,7 +104,7 @@ public interface Cursor extends Inspector { /** * Appends an array entry containing a new value of OBJECT type. - * Returns a valid Cursor (thay may again be used for setting + * Returns a valid Cursor (that may again be used for setting * sub-fields inside the new object) referencing the new entry * value if successful; otherwise returns an invalid Cursor. * @@ -190,7 +190,7 @@ public interface Cursor extends Inspector { /** * Sets a field (identified with a symbol id) to contain a new - * value of ARRAY type. Returns a valid Cursor (thay may again be + * value of ARRAY type. Returns a valid Cursor (that may again be * used for adding new array entries) referencing the new field * value if successful; otherwise returns an invalid Cursor. * @@ -201,7 +201,7 @@ public interface Cursor extends Inspector { /** * Sets a field (identified with a symbol id) to contain a new - * value of OBJECT type. Returns a valid Cursor (thay may again + * value of OBJECT type. Returns a valid Cursor (that may again * be used for setting sub-fields inside the new object) * referencing the new field value if successful; otherwise * returns an invalid Cursor. @@ -289,7 +289,7 @@ public interface Cursor extends Inspector { /** * Sets a field (identified with a symbol name) to contain a new - * value of ARRAY type. Returns a valid Cursor (thay may again be + * value of ARRAY type. Returns a valid Cursor (that may again be * used for adding new array entries) referencing the new field * value if successful; otherwise returns an invalid Cursor. * @@ -300,7 +300,7 @@ public interface Cursor extends Inspector { /** * Sets a field (identified with a symbol name) to contain a new - * value of OBJECT type. Returns a valid Cursor (thay may again + * value of OBJECT type. Returns a valid Cursor (that may again * be used for setting sub-fields inside the new object) * referencing the new field value if successful; otherwise * returns an invalid Cursor. diff --git a/vespajlib/src/main/java/com/yahoo/slime/Inspector.java b/vespajlib/src/main/java/com/yahoo/slime/Inspector.java index bccc8d85223..3d56fc721a2 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Inspector.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Inspector.java @@ -115,7 +115,7 @@ public interface Inspector { Inspector entry(int idx); /** - * Access an field in an object by symbol id. + * Access a field in an object by symbol id. * * If the current Inspector doesn't connect to an object value, or * the object value does not contain a field with the given symbol @@ -126,7 +126,7 @@ public interface Inspector { Inspector field(int sym); /** - * Access an field in an object by symbol name. + * Access a field in an object by symbol name. * * If the current Inspector doesn't connect to an object value, or * the object value does not contain a field with the given symbol diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java index 1ddc9dcfb9e..44cf3e06c20 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java @@ -7,6 +7,7 @@ import com.yahoo.text.Utf8; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.HashMap; +import java.util.function.Supplier; /** * The base class to do cross-language serialization and deserialization of complete object structures without @@ -155,10 +156,9 @@ public class Identifiable extends Selectable implements Cloneable { @Override public boolean equals(Object obj) { - if (!(obj instanceof Identifiable)) { + if (!(obj instanceof Identifiable rhs)) { return false; } - Identifiable rhs = (Identifiable)obj; return (getClassId() == rhs.getClassId()); } @@ -172,7 +172,6 @@ public class Identifiable extends Selectable implements Cloneable { /** * Registers the given class specification for the given identifier in the class registry. This method returns the * supplied identifier, so that subclasses can declare a static classId member like so: - * * <code>public static int classId = registerClass(<id>, <ClassName>.class);</code> * * @param id the class identifier to register with @@ -186,6 +185,14 @@ public class Identifiable extends Selectable implements Cloneable { registry.add(id, spec); return id; } + protected static int registerClass(int id, Class<? extends Identifiable> spec, Supplier<? extends Identifiable> creator) { + if (registry == null) { + registry = new Registry(); + } + registry.add(id, spec, creator); + return id; + } + /** * Deserializes a single {@link Identifiable} object from the given byte buffer. The object itself may perform @@ -282,7 +289,7 @@ public class Identifiable extends Selectable implements Cloneable { private static class Registry { // The map from class id to class descriptor. - private HashMap<Integer, Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>>> typeMap = + private final HashMap<Integer, Pair<Class<? extends Identifiable>, Supplier<? extends Identifiable>>> typeMap = new HashMap<>(); /** @@ -293,18 +300,22 @@ public class Identifiable extends Selectable implements Cloneable { * @throws IllegalArgumentException Thrown if two classes attempt to register with the same identifier. */ private void add(int id, Class<? extends Identifiable> spec) { + + CreateFromConstructor creator; + try { + creator = new CreateFromConstructor(spec.getConstructor()); + } catch (NoSuchMethodException e) { + creator = null; + } + add(id, spec, creator); + } + private void add(int id, Class<? extends Identifiable> spec, Supplier<? extends Identifiable> construct) { Class<?> old = get(id); if (old == null) { - Constructor<? extends Identifiable> constructor; - try { - constructor = spec.getConstructor(); - } catch (NoSuchMethodException e) { - constructor = null; - } - typeMap.put(id, new Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>>(spec, constructor)); + typeMap.put(id, new Pair<Class<? extends Identifiable>, Supplier<? extends Identifiable>>(spec, construct)); } else if (!spec.equals(old)) { - throw new IllegalArgumentException("Can not register class '" + spec.toString() + "' with id " + id + - ", because it already maps to class '" + old.toString() + "'."); + throw new IllegalArgumentException("Can not register class '" + spec + "' with id " + id + + ", because it already maps to class '" + old + "'."); } } @@ -315,42 +326,42 @@ public class Identifiable extends Selectable implements Cloneable { * @return The class specification, may be null. */ private Class<? extends Identifiable> get(int id) { - Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>> pair = typeMap.get(id); + var pair = typeMap.get(id); return (pair != null) ? pair.getFirst() : null; } /** - * Creates an instance of the class mapped to by the given identifier. This method proxies {@link - * #createFromClass(Constructor)}. - * + * Creates an instance of the class mapped to by the given identifier. * @param id The id of the class to create. * @return The instantiated object. */ private Identifiable createFromId(int id) { - Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>> pair = typeMap.get(id); - return createFromClass((pair != null) ? pair.getSecond() : null); + var pair = typeMap.get(id); + return (pair != null) ? pair.getSecond().get() : null; } /** * Creates an instance of a given class specification. All instantiation-type exceptions are consumed and * wrapped inside a runtime exception so that calling methods can let this propagate without declaring them * thrown. - * - * @param spec The class to instantiate. - * @return The instantiated object. - * @throws IllegalArgumentException Thrown if instantiation failed. */ - private Identifiable createFromClass(Constructor<? extends Identifiable> spec) { - Identifiable obj = null; - if (spec != null) { - try { - obj = spec.newInstance(); - } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { - throw new IllegalArgumentException("Failed to create object from class '" + - spec.getName() + "'.", e); + private static class CreateFromConstructor implements Supplier<Identifiable> { + private final Constructor<? extends Identifiable> constructor; + CreateFromConstructor(Constructor<? extends Identifiable> constructor) { + this.constructor = constructor; + } + public Identifiable get() { + Identifiable obj = null; + if (constructor != null) { + try { + obj = constructor.newInstance(); + } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { + throw new IllegalArgumentException("Failed to create object from class '" + + constructor.getName() + "'.", e); + } } + return obj; } - return obj; } } |