diff options
29 files changed, 595 insertions, 71 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 9c35a37c470..0a59392789a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -84,7 +84,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForBucketMove() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default double maxDeadBytesRatio() { return 0.2; } - @ModelFeatureFlag(owners = {"hmusum"}) default int clusterControllerMaxHeapSizeInMb() { return 256; } + @ModelFeatureFlag(owners = {"hmusum"}) default int clusterControllerMaxHeapSizeInMb() { return 128; } @ModelFeatureFlag(owners = {"hmusum"}) default int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return 256; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"tokle"}) default boolean tenantIamRole() { return false; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index c1d45c213fb..306f36e7674 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -8,6 +8,7 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.Quota; +import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.ClusterSpec; @@ -58,6 +59,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int clusterControllerMaxHeapSizeInMb = 256; private int metricsProxyMaxHeapSizeInMb = 256; private int maxActivationInhibitedOutOfSyncGroups = 0; + private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -97,6 +99,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } + @Override public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; } public TestProperties setFeedConcurrency(double feedConcurrency) { this.feedConcurrency = feedConcurrency; @@ -233,6 +236,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setTenantSecretStores(List<TenantSecretStore> secretStores) { + this.tenantSecretStores = List.copyOf(secretStores); + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java index 39f9a627e0c..bf6a0275a94 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java @@ -33,9 +33,9 @@ public class CloudSecretStore extends SimpleComponent implements SecretStoreConf @Override public void getConfig(SecretStoreConfig.Builder builder) { - builder.groups( + builder.awsParameterStores( configList.stream() - .map(config -> new SecretStoreConfig.Groups.Builder() + .map(config -> new SecretStoreConfig.AwsParameterStores.Builder() .name(config.name) .region(config.region) .awsId(config.awsId) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 21aad7a565c..7b3bc498164 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -281,7 +281,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { store -> store )); - for (Element group : XML.getChildren(secretStoreElement, "group")) { + for (Element group : XML.getChildren(secretStoreElement, "aws-parameter-store")) { String name = group.getAttribute("name"); String region = group.getAttribute("region"); TenantSecretStore secretStore = secretStoresByName.get(name); diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index abe7386fa00..9313d91ea55 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -89,9 +89,12 @@ SecretStore = element secret-store { attribute type { string "oath-ckms" | string "cloud" } & element group { attribute name { string } & - (attribute environment { string "alpha" | string "corp" | string "prod" | string "aws" | string "aws_stage" } | - attribute region { string } ) - } + + attribute environment { string "alpha" | string "corp" | string "prod" | string "aws" | string "aws_stage" } + } * & + element aws-parameter-store { + attribute name { string } & + attribute region { string } + } * } ZooKeeper = element zookeeper { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 92e0b116878..e43b5085528 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -15,6 +16,7 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.Cloud; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.RegionName; @@ -29,6 +31,7 @@ import com.yahoo.container.handler.VipStatusHandler; import com.yahoo.container.handler.metrics.MetricsV2Handler; import com.yahoo.container.handler.observability.ApplicationStatusHandler; import com.yahoo.container.jdisc.JdiscBindingsConfig; +import com.yahoo.container.jdisc.secretstore.SecretStoreConfig; import com.yahoo.container.servlet.ServletConfigConfig; import com.yahoo.container.usability.BindingsOverviewHandler; import com.yahoo.jdisc.http.ConnectorConfig; @@ -715,6 +718,54 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + public void cloud_secret_store_requires_configured_secret_store() { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + " <secret-store type='cloud'>", + " <aws-parameter-store name='store1' region='eu-north-1'/>", + " </secret-store>", + "</container>"); + try { + createModel(root, clusterElem); + fail("secret store not defined"); + } catch (RuntimeException e) { + assertEquals("No configured secret store named store1", e.getMessage()); + } + } + + + @Test + public void cloud_secret_store_can_be_set_up() { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + " <secret-store type='cloud'>", + " <aws-parameter-store name='store1' region='eu-north-1'/>", + " </secret-store>", + "</container>"); + + DeployState state = new DeployState.Builder() + .properties( + new TestProperties() + .setHostedVespa(true) + .setTenantSecretStores(List.of(new TenantSecretStore("store1", "1234", "role", Optional.of("externalid"))))) + .zone(new Zone(SystemName.Public, Environment.prod, RegionName.defaultName())) + .build(); + createModel(root, state, null, clusterElem); + + ApplicationContainerCluster container = getContainerCluster("container"); + assertComponentConfigured(container, "com.yahoo.jdisc.cloud.aws.AwsParameterStore"); + CloudSecretStore secretStore = (CloudSecretStore) container.getComponentsMap().get(ComponentId.fromString("com.yahoo.jdisc.cloud.aws.AwsParameterStore")); + + + SecretStoreConfig.Builder configBuilder = new SecretStoreConfig.Builder(); + secretStore.getConfig(configBuilder); + SecretStoreConfig secretStoreConfig = configBuilder.build(); + + assertEquals(1, secretStoreConfig.awsParameterStores().size()); + assertEquals("store1", secretStoreConfig.awsParameterStores().get(0).name()); + } + + @Test public void missing_security_clients_pem_fails_in_public() { Element clusterElem = DomBuilderTest.parse("<container version='1.0' />"); diff --git a/container-disc/abi-spec.json b/container-disc/abi-spec.json index 63f6b241750..735211ff47c 100644 --- a/container-disc/abi-spec.json +++ b/container-disc/abi-spec.json @@ -60,6 +60,43 @@ ], "fields": [] }, + "com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores)", + "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores$Builder name(java.lang.String)", + "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores$Builder region(java.lang.String)", + "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores$Builder awsId(java.lang.String)", + "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores$Builder role(java.lang.String)", + "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores$Builder externalId(java.lang.String)", + "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores build()" + ], + "fields": [] + }, + "com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores$Builder)", + "public java.lang.String name()", + "public java.lang.String region()", + "public java.lang.String awsId()", + "public java.lang.String role()", + "public java.lang.String externalId()" + ], + "fields": [] + }, "com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Builder": { "superClass": "java.lang.Object", "interfaces": [ @@ -73,6 +110,8 @@ "public void <init>(com.yahoo.container.jdisc.secretstore.SecretStoreConfig)", "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Builder groups(com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Groups$Builder)", "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Builder groups(java.util.List)", + "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Builder awsParameterStores(com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores$Builder)", + "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Builder awsParameterStores(java.util.List)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", "public final java.lang.String getDefMd5()", "public final java.lang.String getDefName()", @@ -82,7 +121,8 @@ "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig build()" ], "fields": [ - "public java.util.List groups" + "public java.util.List groups", + "public java.util.List awsParameterStores" ] }, "com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Groups$Builder": { @@ -151,7 +191,9 @@ "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Builder)", "public java.util.List groups()", - "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Groups groups(int)" + "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Groups groups(int)", + "public java.util.List awsParameterStores()", + "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$AwsParameterStores awsParameterStores(int)" ], "fields": [ "public static final java.lang.String CONFIG_DEF_MD5", diff --git a/container-disc/src/main/resources/configdefinitions/container.jdisc.secretstore.secret-store.def b/container-disc/src/main/resources/configdefinitions/container.jdisc.secretstore.secret-store.def index c3edc10c0bc..67ef4744da8 100644 --- a/container-disc/src/main/resources/configdefinitions/container.jdisc.secretstore.secret-store.def +++ b/container-disc/src/main/resources/configdefinitions/container.jdisc.secretstore.secret-store.def @@ -1,8 +1,15 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=container.jdisc.secretstore +# remove groups after 7.376 is oldest model groups[].name string groups[].region string groups[].awsId string groups[].role string -groups[].externalId string
\ No newline at end of file +groups[].externalId string + +awsParameterStores[].name string +awsParameterStores[].region string +awsParameterStores[].awsId string +awsParameterStores[].role string +awsParameterStores[].externalId string diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequest.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequest.java index a56f96955de..31665c8ae0a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequest.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/ChangeRequest.java @@ -13,15 +13,15 @@ public class ChangeRequest { private final ChangeRequestSource changeRequestSource; private final List<String> impactedSwitches; private final List<String> impactedHosts; - private final boolean isApproved; + private final Approval approval; private final Impact impact; - private ChangeRequest(String id, ChangeRequestSource changeRequestSource, List<String> impactedSwitches, List<String> impactedHosts, boolean isApproved, Impact impact) { + private ChangeRequest(String id, ChangeRequestSource changeRequestSource, List<String> impactedSwitches, List<String> impactedHosts, Approval approval, Impact impact) { this.id = Objects.requireNonNull(id); this.changeRequestSource = Objects.requireNonNull(changeRequestSource); this.impactedSwitches = Objects.requireNonNull(impactedSwitches); this.impactedHosts = Objects.requireNonNull(impactedHosts); - this.isApproved = Objects.requireNonNull(isApproved); + this.approval = Objects.requireNonNull(approval); this.impact = Objects.requireNonNull(impact); } @@ -41,8 +41,8 @@ public class ChangeRequest { return impactedHosts; } - public boolean isApproved() { - return isApproved; + public Approval getApproval() { + return approval; } public Impact getImpact() { @@ -56,7 +56,7 @@ public class ChangeRequest { ", changeRequestSource=" + changeRequestSource + ", impactedSwitches=" + impactedSwitches + ", impactedHosts=" + impactedHosts + - ", isApproved=" + isApproved + + ", approval=" + approval + ", impact=" + impact + '}'; } @@ -66,7 +66,7 @@ public class ChangeRequest { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ChangeRequest that = (ChangeRequest) o; - return isApproved == that.isApproved && + return approval == that.approval && Objects.equals(id, that.id) && Objects.equals(changeRequestSource, that.changeRequestSource) && Objects.equals(impactedSwitches, that.impactedSwitches) && @@ -76,7 +76,7 @@ public class ChangeRequest { @Override public int hashCode() { - return Objects.hash(id, changeRequestSource, impactedSwitches, impactedHosts, isApproved, impact); + return Objects.hash(id, changeRequestSource, impactedSwitches, impactedHosts, approval, impact); } public static class Builder { @@ -84,7 +84,7 @@ public class ChangeRequest { private ChangeRequestSource changeRequestSource; private List<String> impactedSwitches; private List<String> impactedHosts; - private boolean isApproved; + private Approval approval; private Impact impact; @@ -108,8 +108,8 @@ public class ChangeRequest { return this; } - public Builder isApproved(boolean isApproved) { - this.isApproved = isApproved; + public Builder approval(Approval approval) { + this.approval = approval; return this; } @@ -119,7 +119,7 @@ public class ChangeRequest { } public ChangeRequest build() { - return new ChangeRequest(id, changeRequestSource, impactedSwitches, impactedHosts, isApproved, impact); + return new ChangeRequest(id, changeRequestSource, impactedSwitches, impactedHosts, approval, impact); } public String getId() { @@ -136,4 +136,11 @@ public class ChangeRequest { UNKNOWN } + public enum Approval { + REQUESTED, + APPROVED, + REJECTED, + OTHER + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/MockChangeRequestClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/MockChangeRequestClient.java new file mode 100644 index 00000000000..e85c0afcb0e --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/MockChangeRequestClient.java @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.vcmr; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author olaa + */ +public class MockChangeRequestClient implements ChangeRequestClient { + + private List<ChangeRequest> upcomingChangeRequests = new ArrayList<>(); + private List<ChangeRequest> approvedChangeRequests = new ArrayList<>(); + + @Override + public List<ChangeRequest> getUpcomingChangeRequests() { + return upcomingChangeRequests; + } + + @Override + public void approveChangeRequests(List<ChangeRequest> changeRequests) { + approvedChangeRequests.addAll(changeRequests); + } + + public void setUpcomingChangeRequests(List<ChangeRequest> changeRequests) { + upcomingChangeRequests = changeRequests; + } + + public List<ChangeRequest> getApprovedChangeRequests() { + return approvedChangeRequests; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/NoopChangeRequestClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/NoopChangeRequestClient.java deleted file mode 100644 index d548620b062..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/NoopChangeRequestClient.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.vcmr; - -import java.util.List; - -/** - * @author olaa - */ -public class NoopChangeRequestClient implements ChangeRequestClient { - - @Override - public List<ChangeRequest> getUpcomingChangeRequests() { - return List.of(); - } - - @Override - public void approveChangeRequests(List<ChangeRequest> changeRequests) {} - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java index 71291e5d766..ca9ebe132fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainer.java @@ -3,9 +3,11 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; import java.time.Duration; +import java.util.List; import java.util.function.Predicate; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -17,10 +19,12 @@ public class ChangeRequestMaintainer extends ControllerMaintainer { private final Logger logger = Logger.getLogger(ChangeRequestMaintainer.class.getName()); private final ChangeRequestClient changeRequestClient; + private final SystemName system; public ChangeRequestMaintainer(Controller controller, Duration interval) { super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.changeRequestClient = controller.serviceRegistry().changeRequestClient(); + this.system = controller.system(); } @@ -29,18 +33,23 @@ public class ChangeRequestMaintainer extends ControllerMaintainer { var changeRequests = changeRequestClient.getUpcomingChangeRequests(); if (!changeRequests.isEmpty()) { - logger.fine(() -> "Found the following upcoming change requests:"); - changeRequests.forEach(changeRequest -> logger.fine(changeRequest::toString)); + logger.info(() -> "Found the following upcoming change requests:"); + changeRequests.forEach(changeRequest -> logger.info(changeRequest::toString)); } + if (system.equals(SystemName.main)) + approveChanges(changeRequests); + + // TODO: Store in curator? + return true; + } + + private void approveChanges(List<ChangeRequest> changeRequests) { var unapprovedRequests = changeRequests .stream() - .filter(changeRequest -> !changeRequest.isApproved()) + .filter(changeRequest -> changeRequest.getApproval() == ChangeRequest.Approval.REQUESTED) .collect(Collectors.toList()); changeRequestClient.approveChangeRequests(unapprovedRequests); - - // TODO: Store in curator? - return true; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index e96eed2b6d9..326928b9463 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportCons import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.secrets.NoopTenantSecretService; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummySystemMonitor; import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; @@ -36,7 +35,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClien import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; -import com.yahoo.vespa.hosted.controller.api.integration.vcmr.NoopChangeRequestClient; +import com.yahoo.vespa.hosted.controller.api.integration.vcmr.MockChangeRequestClient; /** * A mock implementation of a {@link ServiceRegistry} for testing purposes. @@ -72,7 +71,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ContainerRegistryMock containerRegistry = new ContainerRegistryMock(); private final NoopTenantSecretService tenantSecretService = new NoopTenantSecretService(); private final ArchiveService archiveService = new MockArchiveService(); - private final ChangeRequestClient changeRequestClient = new NoopChangeRequestClient(); + private final MockChangeRequestClient changeRequestClient = new MockChangeRequestClient(); public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); @@ -225,7 +224,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public ChangeRequestClient changeRequestClient() { + public MockChangeRequestClient changeRequestClient() { return changeRequestClient; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java new file mode 100644 index 00000000000..b4df16348d2 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeRequestMaintainerTest.java @@ -0,0 +1,59 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; +import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestSource; +import com.yahoo.vespa.hosted.controller.api.integration.vcmr.MockChangeRequestClient; +import org.junit.Test; + +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author olaa + */ +public class ChangeRequestMaintainerTest { + + private final ControllerTester tester = new ControllerTester(); + private final MockChangeRequestClient changeRequestClient = tester.serviceRegistry().changeRequestClient(); + private final ChangeRequestMaintainer changeRequestMaintainer = new ChangeRequestMaintainer(tester.controller(), Duration.ofMinutes(1)); + + @Test + public void only_approve_requests_pending_approval() { + + var upcomingChangeRequests = List.of( + newChangeRequest("id1", ChangeRequest.Approval.APPROVED), + newChangeRequest("id2", ChangeRequest.Approval.REQUESTED) + ); + + changeRequestClient.setUpcomingChangeRequests(upcomingChangeRequests); + changeRequestMaintainer.maintain(); + + var approvedChangeRequests = changeRequestClient.getApprovedChangeRequests(); + + assertEquals(1, approvedChangeRequests.size()); + assertEquals("id2", approvedChangeRequests.get(0).getId()); + } + + private ChangeRequest newChangeRequest(String id, ChangeRequest.Approval approval) { + return new ChangeRequest.Builder() + .id(id) + .approval(approval) + .impact(ChangeRequest.Impact.VERY_HIGH) + .impactedSwitches(List.of()) + .impactedHosts(List.of()) + .changeRequestSource(new ChangeRequestSource.Builder() + .plannedStartTime(ZonedDateTime.now()) + .plannedEndTime(ZonedDateTime.now()) + .id("some-id") + .url("some-url") + .system("some-system") + .status(ChangeRequestSource.Status.CLOSED) + .build()) + .build(); + + } +}
\ No newline at end of file diff --git a/dist/vespa-engine.repo b/dist/vespa-engine.repo index 14e522f6993..33d439a9572 100644 --- a/dist/vespa-engine.repo +++ b/dist/vespa-engine.repo @@ -1,6 +1,8 @@ [vespa-engine-stable] name=vespa-engine-stable -baseurl=https://yahoo.bintray.com/vespa-engine/centos/$releasever/stable/$basearch -gpgcheck=0 +baseurl=https://download.copr.fedorainfracloud.org/results/@vespa/vespa/epel-7-$basearch/ +type=rpm-md +gpgcheck=1 +gpgkey=https://download.copr.fedorainfracloud.org/results/@vespa/vespa/pubkey.gpg repo_gpgcheck=0 enabled=1 diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 478331dd513..38c3bd47e0d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -228,7 +228,7 @@ public class Flags { TENANT_ID, ZONE_ID); public static final UnboundIntFlag CLUSTER_CONTROLLER_MAX_HEAP_SIZE_IN_MB = defineIntFlag( - "cluster-controller-max-heap-size-in-mb", 256, + "cluster-controller-max-heap-size-in-mb", 128, List.of("hmusum"), "2021-02-10", "2021-04-10", "JVM max heap size for cluster controller in Mb", "Takes effect when restarting cluster controller"); diff --git a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java index f2cac68c030..595bb4f6786 100644 --- a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java +++ b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java @@ -82,7 +82,7 @@ public class AwsParameterStore extends AbstractComponent implements SecretStore } private static List<AwsSettings> translateConfig(SecretStoreConfig secretStoreConfig) { - return secretStoreConfig.groups() + return secretStoreConfig.awsParameterStores() .stream() .map(config -> new AwsSettings(config.name(), config.role(), config.awsId(), config.externalId(), config.region())) .collect(Collectors.toList()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java index c8c3cc0daa5..1274e83fb3a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -27,6 +27,8 @@ import java.util.stream.Collectors; */ public class RetiredExpirer extends NodeRepositoryMaintainer { + private static final int NUM_CONFIG_SERVERS = 3; + private final Deployer deployer; private final Metric metric; private final Orchestrator orchestrator; @@ -57,7 +59,7 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { for (Map.Entry<ApplicationId, List<Node>> entry : retiredNodesByApplication.entrySet()) { ApplicationId application = entry.getKey(); List<Node> retiredNodes = entry.getValue(); - List<Node> nodesToRemove = retiredNodes.stream().filter(this::canRemove).collect(Collectors.toList()); + List<Node> nodesToRemove = retiredNodes.stream().filter(n -> canRemove(n, activeNodes)).collect(Collectors.toList()); if (nodesToRemove.isEmpty()) continue; try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) { @@ -81,7 +83,7 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { * - The node has been in state {@link History.Event.Type#retired} for longer than {@link #retiredExpiry} * - Orchestrator allows it */ - private boolean canRemove(Node node) { + private boolean canRemove(Node node, NodeList activeNodes) { if (node.type().isHost()) { if (nodeRepository().nodes().list().childrenOf(node).asList().stream() .allMatch(child -> child.state() == Node.State.parked || @@ -93,7 +95,32 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { return false; } - if (node.history().hasEventBefore(History.Event.Type.retired, clock().instant().minus(retiredExpiry))) { + if (node.type().isConfigServerLike()) { + // Avoid eventual expiry of configserver-like nodes + + if (activeNodes.nodeType(node.type()).size() < NUM_CONFIG_SERVERS) { + // Scenario: All 3 config servers want to retire. + // + // Say RetiredExpirer runs on cfg1 and gives cfg2 permission to be removed (PERMANENTLY_DOWN in ZK). + // The consequent redeployment moves cfg2 to inactive, removing cfg2 from the application, + // and PERMANENTLY_DOWN for cfg2 is cleaned up. + // + // If the RetiredExpirer on cfg3 now runs before its InfrastructureProvisioner, then + // a. The duper model still contains cfg2 + // b. The service model still monitors cfg2 for health and it is UP + // c. The Orchestrator has no host status (like PERMANENTLY_DOWN) for cfg2, + // which is equivalent to NO_REMARKS + // Therefore, from the point of view of the Orchestrator invoked below, any cfg will + // be allowed to be removed, say cfg1. In the subsequent redeployment, both cfg2 + // and cfg1 are now inactive. + // + // A proper solution would be to ensure the duper model is changed atomically + // with node states across all config servers. As this would require some work, + // we will instead verify here that there are 3 active config servers before + // allowing the removal of any config server. + return false; + } + } else if (node.history().hasEventBefore(History.Event.Type.retired, clock().instant().minus(retiredExpiry))) { log.warning("Node " + node + " has been retired longer than " + retiredExpiry + ": Allowing removal. This may cause data loss"); return true; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java index e7ebf049e51..a57bcce7db4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java @@ -26,6 +26,11 @@ public class MockDuperModel implements DuperModelInfraApi { public MockDuperModel() { } + public MockDuperModel support(InfraApplicationApi infraApplication) { + supportedInfraApps.put(infraApplication.getApplicationId(), infraApplication); + return this; + } + @Override public List<InfraApplicationApi> getSupportedInfraApplications() { return new ArrayList<>(supportedInfraApps.values()); @@ -41,6 +46,10 @@ public class MockDuperModel implements DuperModelInfraApi { return activeApps.containsKey(applicationId); } + public List<HostName> hostnamesOf(ApplicationId applicationId) { + return activeApps.getOrDefault(applicationId, List.of()); + } + @Override public void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames) { activeApps.put(applicationId, hostnames); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index c1c6e5b6154..718facd477c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -1,39 +1,57 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.yahoo.config.provision.ActivationContext; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.IP; +import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; +import com.yahoo.vespa.hosted.provision.provisioning.InfraDeployerImpl; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; +import com.yahoo.vespa.hosted.provision.testutils.MockDuperModel; +import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.vespa.service.duper.ConfigServerApplication; import org.junit.Before; import org.junit.Test; import java.time.Duration; +import java.time.Instant; import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * @author bratseth @@ -152,6 +170,104 @@ public class RetiredExpirerTest { assertFalse(node.allocation().get().membership().retired()); } + @Test + public void config_server_reprovisioning() throws OrchestrationException { + NodeList configServers = tester.makeConfigServers(3, "default", Version.emptyVersion); + var cfg1 = new HostName("cfg1"); + assertEquals(Set.of(cfg1.s(), "cfg2", "cfg3"), configServers.stream().map(Node::hostname).collect(Collectors.toSet())); + + var configServerApplication = new ConfigServerApplication(); + var duperModel = new MockDuperModel().support(configServerApplication); + InfraDeployerImpl infraDeployer = new InfraDeployerImpl(tester.nodeRepository(), tester.provisioner(), duperModel); + + var deployer = mock(Deployer.class); + when(deployer.deployFromLocalActive(eq(configServerApplication.getApplicationId()))) + .thenAnswer(invocation -> infraDeployer.getDeployment(configServerApplication.getApplicationId())); + + // Set wantToRetire on all 3 config servers + List<Node> wantToRetireNodes = tester.nodeRepository().nodes() + .retire(NodeTypeFilter.from(NodeType.config, null), Agent.operator, Instant.now()); + assertEquals(3, wantToRetireNodes.size()); + + // Redeploy to retire all 3 config servers + infraDeployer.activateAllSupportedInfraApplications(true); + + // Only 1 config server is allowed to retire at any given point in time + List<Node> retiredNodes = tester.nodeRepository().nodes().list(() -> {}).stream() + .filter(node -> node.allocation().map(allocation -> allocation.membership().retired()).orElse(false)) + .collect(Collectors.toList()); + assertEquals(1, retiredNodes.size()); + Node retiredNode = retiredNodes.get(0); + String retiredNodeHostname = retiredNode.hostname(); + + // Allow retiredNodeHostname to be removed + doThrow(new OrchestrationException("denied")).when(orchestrator).acquirePermissionToRemove(any()); + doNothing().when(orchestrator).acquirePermissionToRemove(eq(new HostName(retiredNodeHostname))); + + // RetiredExpirer should remove cfg1 from application + RetiredExpirer retiredExpirer = createRetiredExpirer(deployer); + retiredExpirer.run(); + var activeConfigServerHostnames = new HashSet<>(Set.of("cfg1", "cfg2", "cfg3")); + assertTrue(activeConfigServerHostnames.contains(retiredNodeHostname)); + activeConfigServerHostnames.remove(retiredNodeHostname); + assertEquals(activeConfigServerHostnames, configServerHostnames(duperModel)); + assertEquals(1, tester.nodeRepository().nodes().list(Node.State.inactive).nodeType(NodeType.config).size()); + assertEquals(2, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.config).size()); + + // no changes while 1 cfg is inactive + retiredExpirer.run(); + assertEquals(activeConfigServerHostnames, configServerHostnames(duperModel)); + assertEquals(1, tester.nodeRepository().nodes().list(Node.State.inactive).nodeType(NodeType.config).size()); + assertEquals(2, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.config).size()); + + // The node will eventually expire from inactive, and be removed by DynamicProvisioningMaintainer + // (depending on its host), and these events should not affect the 2 active config servers. + nodeRepository.nodes().deallocate(retiredNode, Agent.InactiveExpirer, "expired"); + retiredNode = tester.nodeRepository().nodes().list(Node.State.parked).nodeType(NodeType.config).asList().get(0); + nodeRepository.nodes().removeRecursively(retiredNode, true); + infraDeployer.activateAllSupportedInfraApplications(true); + retiredExpirer.run(); + assertEquals(activeConfigServerHostnames, configServerHostnames(duperModel)); + assertEquals(2, tester.nodeRepository().nodes().list().nodeType(NodeType.config).size()); + assertEquals(2, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.config).size()); + + // Provision and ready new config server + MockNameResolver nameResolver = (MockNameResolver)tester.nodeRepository().nameResolver(); + String ipv4 = "127.0.1.4"; + nameResolver.addRecord(retiredNodeHostname, ipv4); + Node node = Node.create(retiredNodeHostname, new IP.Config(Set.of(ipv4), Set.of()), retiredNodeHostname, + tester.asFlavor("default", NodeType.config), NodeType.config).build(); + var nodes = List.of(node); + nodes = nodeRepository.nodes().addNodes(nodes, Agent.system); + nodes = nodeRepository.nodes().deallocate(nodes, Agent.system, getClass().getSimpleName()); + nodeRepository.nodes().setReady(nodes, Agent.system, getClass().getSimpleName()); + + // no changes while replacement config server is ready + retiredExpirer.run(); + assertEquals(activeConfigServerHostnames, configServerHostnames(duperModel)); + assertEquals(1, tester.nodeRepository().nodes().list(Node.State.ready).nodeType(NodeType.config).size()); + assertEquals(2, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.config).size()); + + // Activate replacement config server + infraDeployer.activateAllSupportedInfraApplications(true); + assertEquals(3, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.config).size()); + + // Another config server should now have retired + retiredExpirer.run(); + assertEquals(3, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.config).size()); + var retiredNodes2 = tester.nodeRepository().nodes().list(() -> {}).stream() + .filter(n -> n.allocation().map(allocation -> allocation.membership().retired()).orElse(false)) + .collect(Collectors.toList()); + assertEquals(1, retiredNodes2.size()); + assertNotEquals(retiredNodeHostname, retiredNodes2.get(0)); + } + + private Set<String> configServerHostnames(MockDuperModel duperModel) { + return duperModel.hostnamesOf(new ConfigServerApplication().getApplicationId()).stream() + .map(com.yahoo.config.provision.HostName::value) + .collect(Collectors.toSet()); + } + private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodes, int groups) { Capacity capacity = Capacity.from(new ClusterResources(nodes, groups, nodeResources)); tester.activate(applicationId, tester.prepare(applicationId, cluster, capacity)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index a30735df01c..3e71b7f5158 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -490,7 +490,7 @@ public class ProvisioningTester { return nodeRepository.nodes().setReady(nodes, Agent.system, getClass().getSimpleName()); } - private Flavor asFlavor(String flavorString, NodeType type) { + public Flavor asFlavor(String flavorString, NodeType type) { Optional<Flavor> flavor = nodeFlavors.getFlavor(flavorString); if (flavor.isEmpty()) { // TODO: Remove the need for this by always adding hosts with a given capacity diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index d412f408350..9e958dd4d4c 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -897,6 +897,7 @@ "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorXwPlusB()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorArgmax()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorArgmin()", + "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorCellCast()", "public final com.yahoo.searchlib.rankingexpression.rule.LambdaFunctionNode lambdaFunction()", "public final com.yahoo.tensor.functions.Reduce$Aggregator tensorReduceAggregator()", "public final com.yahoo.tensor.TensorType tensorType(java.util.List)", @@ -1046,6 +1047,7 @@ "public static final int XW_PLUS_B", "public static final int ARGMAX", "public static final int ARGMIN", + "public static final int CELL_CAST", "public static final int AVG", "public static final int COUNT", "public static final int MAX", diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj index 36b1f9627bb..7506fe250fc 100755 --- a/searchlib/src/main/javacc/RankingExpressionParser.jj +++ b/searchlib/src/main/javacc/RankingExpressionParser.jj @@ -141,6 +141,7 @@ TOKEN : <XW_PLUS_B: "xw_plus_b"> | <ARGMAX: "argmax"> | <ARGMIN: "argmin"> | + <CELL_CAST: "cell_cast"> | <AVG: "avg" > | <COUNT: "count"> | @@ -380,7 +381,8 @@ TensorFunctionNode tensorFunction() : tensorExpression = tensorSoftmax() | tensorExpression = tensorXwPlusB() | tensorExpression = tensorArgmax() | - tensorExpression = tensorArgmin() + tensorExpression = tensorArgmin() | + tensorExpression = tensorCellCast() ) { return tensorExpression; } } @@ -597,6 +599,16 @@ TensorFunctionNode tensorArgmin() : { return new TensorFunctionNode(new Argmin(TensorFunctionNode.wrap(tensor), dimensions)); } } +TensorFunctionNode tensorCellCast() : +{ + ExpressionNode tensor; + String valueType; +} +{ + <CELL_CAST> <LBRACE> tensor = expression() <COMMA> valueType = identifier() <RBRACE> + { return new TensorFunctionNode(new CellCast(TensorFunctionNode.wrap(tensor), TensorType.Value.fromId(valueType)));} +} + LambdaFunctionNode lambdaFunction() : { List<String> variables; @@ -667,7 +679,7 @@ String tensorFunctionName() : ( <MAP> { return token.image; } ) | ( <REDUCE> { return token.image; } ) | ( <JOIN> { return token.image; } ) | - ( <MERGE> { return token.image; } ) | + ( <MERGE> { return token.image; } ) | ( <RENAME> { return token.image; } ) | ( <CONCAT> { return token.image; } ) | ( <TENSOR> { return token.image; } ) | @@ -681,6 +693,7 @@ String tensorFunctionName() : ( <XW_PLUS_B> { return token.image; } ) | ( <ARGMAX> { return token.image; } ) | ( <ARGMIN> { return token.image; } ) | + ( <CELL_CAST> { return token.image; } ) | ( aggregator = tensorReduceAggregator() { return aggregator.toString(); } ) } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 123fa5ac43b..5c4840a555d 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -394,6 +394,22 @@ public class EvaluationTestCase { } @Test + public void testCellTypeCasting() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor<float>(x[3]):[1.0, 2.0, 3.0]", + "cell_cast(tensor0, float)", + "tensor<double>(x[3]):[1, 2, 3]"); + tester.assertEvaluates("tensor<float>():{1}", + "cell_cast(tensor0{x:1}, float)", + "tensor<double>(x{}):{1:1, 2:2, 3:3}"); + tester.assertEvaluates("tensor<float>(x[2]):[3,8]", + "cell_cast(tensor0 * tensor1, float)", + "tensor<float>(x[2]):[1,2]", + "tensor<double>(x[2]):[3,4]"); + } + + @Test public void testMixedTensorType() throws ParseException { String expected = "tensor(x[1],y{},z[2]):{{x:0,y:a,z:0}:4.0,{x:0,y:a,z:1}:5.0,{x:0,y:b,z:0}:7.0,{x:0,y:b,z:1}:8.0}"; String a = "tensor(x[1],y{}):{ {x:0,y:a}:1, {x:0,y:b}:2 }"; diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index c6727aa372e..e51569da988 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1167,6 +1167,7 @@ "public com.yahoo.tensor.Tensor concat(com.yahoo.tensor.Tensor, java.lang.String)", "public com.yahoo.tensor.Tensor rename(java.util.List, java.util.List)", "public static com.yahoo.tensor.Tensor generate(com.yahoo.tensor.TensorType, java.util.function.Function)", + "public com.yahoo.tensor.Tensor cellCast(com.yahoo.tensor.TensorType$Value)", "public com.yahoo.tensor.Tensor l1Normalize(java.lang.String)", "public com.yahoo.tensor.Tensor l2Normalize(java.lang.String)", "public com.yahoo.tensor.Tensor matmul(com.yahoo.tensor.Tensor, java.lang.String)", @@ -1569,6 +1570,23 @@ ], "fields": [] }, + "com.yahoo.tensor.functions.CellCast": { + "superClass": "com.yahoo.tensor.functions.PrimitiveTensorFunction", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.tensor.functions.TensorFunction, com.yahoo.tensor.TensorType$Value)", + "public java.util.List arguments()", + "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", + "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", + "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", + "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + ], + "fields": [] + }, "com.yahoo.tensor.functions.CompositeTensorFunction": { "superClass": "com.yahoo.tensor.functions.TensorFunction", "interfaces": [], @@ -3445,4 +3463,4 @@ ], "fields": [] } -}
\ No newline at end of file +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index fbf5bc35129..3378520dc91 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -4,6 +4,7 @@ package com.yahoo.tensor; import com.yahoo.tensor.evaluation.TypeContext; import com.yahoo.tensor.functions.Argmax; import com.yahoo.tensor.functions.Argmin; +import com.yahoo.tensor.functions.CellCast; import com.yahoo.tensor.functions.Concat; import com.yahoo.tensor.functions.ConstantTensor; import com.yahoo.tensor.functions.Diag; @@ -179,6 +180,10 @@ public interface Tensor { return new Generate<>(type, valueSupplier).evaluate(); } + default Tensor cellCast(TensorType.Value valueType) { + return new CellCast<>(new ConstantTensor<>(this), valueType).evaluate(); + } + // ----------------- Composite tensor functions which have a defined primitive mapping default Tensor l1Normalize(String dimension) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java new file mode 100644 index 00000000000..d052e383c85 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java @@ -0,0 +1,83 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.functions; + +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.EvaluationContext; +import com.yahoo.tensor.evaluation.Name; +import com.yahoo.tensor.evaluation.TypeContext; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +/** + * The <i>cell_cast</i> tensor function creates a new tensor with the specified cell value type. + * + * @author lesters + */ +public class CellCast<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYPE> { + + private final TensorFunction<NAMETYPE> argument; + private final TensorType.Value valueType; + + public CellCast(TensorFunction<NAMETYPE> argument, TensorType.Value valueType) { + Objects.requireNonNull(argument, "The argument tensor cannot be null"); + Objects.requireNonNull(valueType, "The value type cannot be null"); + this.argument = argument; + this.valueType = valueType; + } + + @Override + public List<TensorFunction<NAMETYPE>> arguments() { return Collections.singletonList(argument); } + + @Override + public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { + if ( arguments.size() != 1) + throw new IllegalArgumentException("CellCast must have 1 argument, got " + arguments.size()); + return new CellCast<>(arguments.get(0), valueType); + } + + @Override + public PrimitiveTensorFunction<NAMETYPE> toPrimitive() { + return new CellCast<>(argument.toPrimitive(), valueType); + } + + @Override + public TensorType type(TypeContext<NAMETYPE> context) { + return new TensorType(valueType, argument.type(context).dimensions()); + } + + @Override + public Tensor evaluate(EvaluationContext<NAMETYPE> context) { + Tensor tensor = argument.evaluate(context); + if (tensor.type().valueType() == valueType) { + return tensor; + } + TensorType type = new TensorType(valueType, tensor.type().dimensions()); + return cast(tensor, type); + } + + private Tensor cast(Tensor tensor, TensorType type) { + Tensor.Builder builder = Tensor.Builder.of(type); + TensorType.Value fromValueType = tensor.type().valueType(); + for (Iterator<Tensor.Cell> i = tensor.cellIterator(); i.hasNext(); ) { + Tensor.Cell cell = i.next(); + if (fromValueType == TensorType.Value.FLOAT) { + builder.cell(cell.getKey(), cell.getFloatValue()); + } else if (fromValueType == TensorType.Value.DOUBLE) { + builder.cell(cell.getKey(), cell.getDoubleValue()); + } else { + builder.cell(cell.getKey(), cell.getValue()); + } + } + return builder.build(); + } + + @Override + public String toString(ToStringContext context) { + return "cell_cast(" + argument.toString(context) + ", " + valueType + ")"; + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/functions/CellCastTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/functions/CellCastTestCase.java new file mode 100644 index 00000000000..bc10ecc3abd --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/tensor/functions/CellCastTestCase.java @@ -0,0 +1,38 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.functions; + +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author lesters + */ +public class CellCastTestCase { + + @Test + public void testCellCasting() { + Tensor tensor; + + tensor = Tensor.from("tensor(x[3]):[1.0, 2.0, 3.0]"); + assertEquals(TensorType.Value.DOUBLE, tensor.type().valueType()); + assertEquals(TensorType.Value.DOUBLE, tensor.cellCast(TensorType.Value.DOUBLE).type().valueType()); + assertEquals(TensorType.Value.FLOAT, tensor.cellCast(TensorType.Value.FLOAT).type().valueType()); + assertEquals(tensor, tensor.cellCast(TensorType.Value.FLOAT)); + + tensor = Tensor.from("tensor<double>(x{}):{{x:0}:1.0,{x:1}:2.0,{x:2}:3.0}"); + assertEquals(TensorType.Value.DOUBLE, tensor.type().valueType()); + assertEquals(TensorType.Value.DOUBLE, tensor.cellCast(TensorType.Value.DOUBLE).type().valueType()); + assertEquals(TensorType.Value.FLOAT, tensor.cellCast(TensorType.Value.FLOAT).type().valueType()); + assertEquals(tensor, tensor.cellCast(TensorType.Value.FLOAT)); + + tensor = Tensor.from("tensor<float>(x[3],y{}):{a:[1.0, 2.0, 3.0],b:[4.0,5.0,6.0]}"); + assertEquals(TensorType.Value.FLOAT, tensor.type().valueType()); + assertEquals(TensorType.Value.DOUBLE, tensor.cellCast(TensorType.Value.DOUBLE).type().valueType()); + assertEquals(TensorType.Value.FLOAT, tensor.cellCast(TensorType.Value.FLOAT).type().valueType()); + assertEquals(tensor, tensor.cellCast(TensorType.Value.DOUBLE)); + } + +} diff --git a/zookeeper-command-line-client/src/main/sh/vespa-zkcli b/zookeeper-command-line-client/src/main/sh/vespa-zkcli index 1a5858b3222..0406e3d89c2 100755 --- a/zookeeper-command-line-client/src/main/sh/vespa-zkcli +++ b/zookeeper-command-line-client/src/main/sh/vespa-zkcli @@ -80,19 +80,15 @@ usage() { echo "" echo "-h|-help) print this help text" - echo "-nosudo do not use sudo when running command" } -sudo="/usr/bin/sudo -u ${VESPA_USER}" while [ $# -gt 0 ]; do case $1 in -h|-help) usage; exit 0;; - -nosudo) shift; sudo="" ;; *) echo "Unrecognized option '$1'" >&2; exit 1;; esac done -$sudo java \ - -cp $VESPA_HOME/lib/jars/zookeeper-command-line-client-jar-with-dependencies.jar \ - -Dlog4j.configuration="log4j-vespa.properties" -Xms32m -Xmx512m \ - com.yahoo.vespa.zookeeper.cli.Main "$@" +java -cp $VESPA_HOME/lib/jars/zookeeper-command-line-client-jar-with-dependencies.jar \ + -Dlog4j.configuration="log4j-vespa.properties" -Xms32m -Xmx512m \ + com.yahoo.vespa.zookeeper.cli.Main "$@" |