diff options
20 files changed, 181 insertions, 969 deletions
diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 853fe4b11ed..9e26dfeeb6e 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -1,875 +1 @@ -{ - "com.yahoo.config.provision.AllocatedHosts": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public static com.yahoo.config.provision.AllocatedHosts withHosts(java.util.Set)", - "public java.util.Set getHosts()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.config.provision.ApplicationId$Builder": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public com.yahoo.config.provision.ApplicationId$Builder tenant(com.yahoo.config.provision.TenantName)", - "public com.yahoo.config.provision.ApplicationId$Builder tenant(java.lang.String)", - "public com.yahoo.config.provision.ApplicationId$Builder applicationName(com.yahoo.config.provision.ApplicationName)", - "public com.yahoo.config.provision.ApplicationId$Builder applicationName(java.lang.String)", - "public com.yahoo.config.provision.ApplicationId$Builder instanceName(com.yahoo.config.provision.InstanceName)", - "public com.yahoo.config.provision.ApplicationId$Builder instanceName(java.lang.String)", - "public com.yahoo.config.provision.ApplicationId build()" - ], - "fields": [] - }, - "com.yahoo.config.provision.ApplicationId": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.Comparable" - ], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.cloud.config.ApplicationIdConfig)", - "public static com.yahoo.config.provision.ApplicationId from(com.yahoo.config.provision.TenantName, com.yahoo.config.provision.ApplicationName, com.yahoo.config.provision.InstanceName)", - "public static com.yahoo.config.provision.ApplicationId from(java.lang.String, java.lang.String, java.lang.String)", - "public static com.yahoo.config.provision.ApplicationId fromSerializedForm(java.lang.String)", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public java.lang.String serializedForm()", - "public java.lang.String toShortString()", - "public java.lang.String toFullString()", - "public java.lang.String toString()", - "public com.yahoo.config.provision.TenantName tenant()", - "public com.yahoo.config.provision.ApplicationName application()", - "public com.yahoo.config.provision.InstanceName instance()", - "public int compareTo(com.yahoo.config.provision.ApplicationId)", - "public static com.yahoo.config.provision.ApplicationId defaultId()", - "public static com.yahoo.config.provision.ApplicationId global()", - "public bridge synthetic int compareTo(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.config.provision.ApplicationLockException": { - "superClass": "java.lang.RuntimeException", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.Exception)" - ], - "fields": [] - }, - "com.yahoo.config.provision.ApplicationName": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.Comparable" - ], - "attributes": [ - "public" - ], - "methods": [ - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public java.lang.String toString()", - "public static com.yahoo.config.provision.ApplicationName from(java.lang.String)", - "public static com.yahoo.config.provision.ApplicationName defaultName()", - "public boolean isDefault()", - "public java.lang.String value()", - "public int compareTo(com.yahoo.config.provision.ApplicationName)", - "public bridge synthetic int compareTo(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.config.provision.AthenzDomain": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public static com.yahoo.config.provision.AthenzDomain from(java.lang.String)", - "public java.lang.String value()", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - }, - "com.yahoo.config.provision.AthenzService": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public java.lang.String value()", - "public static com.yahoo.config.provision.AthenzService from(java.lang.String)", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - }, - "com.yahoo.config.provision.Capacity": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public int nodeCount()", - "public java.util.Optional flavor()", - "public java.util.Optional nodeResources()", - "public boolean isRequired()", - "public boolean canFail()", - "public com.yahoo.config.provision.NodeType type()", - "public java.lang.String toString()", - "public static com.yahoo.config.provision.Capacity fromNodeCount(int)", - "public static com.yahoo.config.provision.Capacity fromCount(int, com.yahoo.config.provision.NodeResources)", - "public static com.yahoo.config.provision.Capacity fromCount(int, com.yahoo.config.provision.NodeResources, boolean, boolean)", - "public static com.yahoo.config.provision.Capacity fromCount(int, java.util.Optional, boolean, boolean)", - "public static com.yahoo.config.provision.Capacity fromRequiredNodeType(com.yahoo.config.provision.NodeType)" - ], - "fields": [] - }, - "com.yahoo.config.provision.CertificateNotReadyException": { - "superClass": "com.yahoo.config.provision.TransientException", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.config.provision.CloudName": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.Comparable" - ], - "attributes": [ - "public" - ], - "methods": [ - "public java.lang.String value()", - "public boolean isDefault()", - "public static com.yahoo.config.provision.CloudName defaultName()", - "public static com.yahoo.config.provision.CloudName from(java.lang.String)", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()", - "public int compareTo(com.yahoo.config.provision.CloudName)", - "public bridge synthetic int compareTo(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.config.provision.ClusterMembership": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "protected void <init>()", - "protected java.lang.String toStringValue()", - "public com.yahoo.config.provision.ClusterSpec cluster()", - "public int index()", - "public boolean retired()", - "public com.yahoo.config.provision.ClusterMembership retire()", - "public com.yahoo.config.provision.ClusterMembership unretire()", - "public com.yahoo.config.provision.ClusterMembership with(com.yahoo.config.provision.ClusterSpec)", - "public java.lang.String stringValue()", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public java.lang.String toString()", - "public static com.yahoo.config.provision.ClusterMembership from(java.lang.String, com.yahoo.component.Version)", - "public static com.yahoo.config.provision.ClusterMembership from(java.lang.String, com.yahoo.component.Version, java.util.Optional)", - "public static com.yahoo.config.provision.ClusterMembership from(com.yahoo.config.provision.ClusterSpec, int)", - "public static com.yahoo.config.provision.ClusterMembership retiredFrom(com.yahoo.config.provision.ClusterSpec, int)" - ], - "fields": [] - }, - "com.yahoo.config.provision.ClusterSpec$Builder": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public com.yahoo.config.provision.ClusterSpec build()", - "public com.yahoo.config.provision.ClusterSpec$Builder group(com.yahoo.config.provision.ClusterSpec$Group)", - "public com.yahoo.config.provision.ClusterSpec$Builder vespaVersion(com.yahoo.component.Version)", - "public com.yahoo.config.provision.ClusterSpec$Builder vespaVersion(java.lang.String)", - "public com.yahoo.config.provision.ClusterSpec$Builder exclusive(boolean)", - "public com.yahoo.config.provision.ClusterSpec$Builder combinedId(java.util.Optional)", - "public com.yahoo.config.provision.ClusterSpec$Builder dockerImageRepo(java.util.Optional)" - ], - "fields": [] - }, - "com.yahoo.config.provision.ClusterSpec$Group": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public static com.yahoo.config.provision.ClusterSpec$Group from(int)", - "public int index()", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - }, - "com.yahoo.config.provision.ClusterSpec$Id": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(java.lang.String)", - "public static com.yahoo.config.provision.ClusterSpec$Id from(java.lang.String)", - "public java.lang.String value()", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - }, - "com.yahoo.config.provision.ClusterSpec$Type": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.provision.ClusterSpec$Type[] values()", - "public static com.yahoo.config.provision.ClusterSpec$Type valueOf(java.lang.String)", - "public boolean isContent()", - "public boolean isContainer()", - "public static com.yahoo.config.provision.ClusterSpec$Type from(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.config.provision.ClusterSpec$Type admin", - "public static final enum com.yahoo.config.provision.ClusterSpec$Type container", - "public static final enum com.yahoo.config.provision.ClusterSpec$Type content", - "public static final enum com.yahoo.config.provision.ClusterSpec$Type combined" - ] - }, - "com.yahoo.config.provision.ClusterSpec": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public com.yahoo.config.provision.ClusterSpec$Type type()", - "public com.yahoo.config.provision.ClusterSpec$Id id()", - "public java.util.Optional dockerImageRepo()", - "public java.util.Optional dockerImage()", - "public com.yahoo.component.Version vespaVersion()", - "public java.util.Optional group()", - "public java.util.Optional combinedId()", - "public boolean isExclusive()", - "public com.yahoo.config.provision.ClusterSpec with(java.util.Optional)", - "public com.yahoo.config.provision.ClusterSpec exclusive(boolean)", - "public static com.yahoo.config.provision.ClusterSpec request(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.component.Version, boolean, java.util.Optional)", - "public static com.yahoo.config.provision.ClusterSpec request(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.component.Version, boolean, java.util.Optional, java.util.Optional)", - "public static com.yahoo.config.provision.ClusterSpec from(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.config.provision.ClusterSpec$Group, com.yahoo.component.Version, boolean, java.util.Optional)", - "public static com.yahoo.config.provision.ClusterSpec from(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.config.provision.ClusterSpec$Group, com.yahoo.component.Version, boolean, java.util.Optional, java.util.Optional)", - "public static com.yahoo.config.provision.ClusterSpec$Builder request(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id)", - "public static com.yahoo.config.provision.ClusterSpec$Builder specification(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id)", - "public java.lang.String toString()", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public boolean satisfies(com.yahoo.config.provision.ClusterSpec)" - ], - "fields": [] - }, - "com.yahoo.config.provision.Deployer": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public java.util.Optional deployFromLocalActive(com.yahoo.config.provision.ApplicationId)", - "public abstract java.util.Optional deployFromLocalActive(com.yahoo.config.provision.ApplicationId, boolean)", - "public java.util.Optional deployFromLocalActive(com.yahoo.config.provision.ApplicationId, java.time.Duration)", - "public abstract java.util.Optional deployFromLocalActive(com.yahoo.config.provision.ApplicationId, java.time.Duration, boolean)", - "public abstract java.util.Optional lastDeployTime(com.yahoo.config.provision.ApplicationId)" - ], - "fields": [] - }, - "com.yahoo.config.provision.Deployment": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void prepare()", - "public abstract void activate()", - "public abstract void restart(com.yahoo.config.provision.HostFilter)" - ], - "fields": [] - }, - "com.yahoo.config.provision.DockerImage": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public java.lang.String repository()", - "public java.util.Optional tag()", - "public com.yahoo.component.Version tagAsVersion()", - "public com.yahoo.config.provision.DockerImage withTag(com.yahoo.component.Version)", - "public java.lang.String asString()", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public static com.yahoo.config.provision.DockerImage fromString(java.lang.String)" - ], - "fields": [ - "public static final com.yahoo.config.provision.DockerImage EMPTY" - ] - }, - "com.yahoo.config.provision.Environment": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.provision.Environment[] values()", - "public static com.yahoo.config.provision.Environment valueOf(java.lang.String)", - "public boolean isManuallyDeployed()", - "public boolean isTest()", - "public boolean isProduction()", - "public boolean isMultiRegion()", - "public static com.yahoo.config.provision.Environment defaultEnvironment()", - "public static com.yahoo.config.provision.Environment from(java.lang.String)", - "public java.lang.String value()" - ], - "fields": [ - "public static final enum com.yahoo.config.provision.Environment prod", - "public static final enum com.yahoo.config.provision.Environment staging", - "public static final enum com.yahoo.config.provision.Environment test", - "public static final enum com.yahoo.config.provision.Environment dev", - "public static final enum com.yahoo.config.provision.Environment perf" - ] - }, - "com.yahoo.config.provision.Flavor$Type": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.provision.Flavor$Type[] values()", - "public static com.yahoo.config.provision.Flavor$Type valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.config.provision.Flavor$Type undefined", - "public static final enum com.yahoo.config.provision.Flavor$Type BARE_METAL", - "public static final enum com.yahoo.config.provision.Flavor$Type VIRTUAL_MACHINE", - "public static final enum com.yahoo.config.provision.Flavor$Type DOCKER_CONTAINER" - ] - }, - "com.yahoo.config.provision.Flavor": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.config.provisioning.FlavorsConfig$Flavor)", - "public void <init>(com.yahoo.config.provision.NodeResources)", - "public void <init>(java.lang.String, com.yahoo.config.provision.NodeResources)", - "public com.yahoo.config.provision.Flavor with(com.yahoo.config.provision.host.FlavorOverrides)", - "public com.yahoo.config.provision.Flavor with(com.yahoo.config.provision.NodeResources)", - "public java.lang.String name()", - "public int cost()", - "public boolean isConfigured()", - "public com.yahoo.config.provision.NodeResources resources()", - "public java.util.Optional flavorOverrides()", - "public double getMinMainMemoryAvailableGb()", - "public double getMinDiskAvailableGb()", - "public boolean hasFastDisk()", - "public double getBandwidthGbps()", - "public double getMinCpuCores()", - "public com.yahoo.config.provision.Flavor$Type getType()", - "public boolean isDocker()", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.config.provision.HostFilter": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public boolean matches(java.lang.String, java.lang.String, java.util.Optional)", - "public static com.yahoo.config.provision.HostFilter all()", - "public static com.yahoo.config.provision.HostFilter hostname(java.lang.String)", - "public static com.yahoo.config.provision.HostFilter flavor(java.lang.String)", - "public static com.yahoo.config.provision.HostFilter clusterType(com.yahoo.config.provision.ClusterSpec$Type)", - "public static com.yahoo.config.provision.HostFilter clusterId(com.yahoo.config.provision.ClusterSpec$Id)", - "public static com.yahoo.config.provision.HostFilter from(java.util.Collection, java.util.Collection, java.util.Collection, java.util.Collection)", - "public static com.yahoo.config.provision.HostFilter from(java.lang.String, java.lang.String, java.lang.String, java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.config.provision.HostLivenessTracker": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void receivedRequestFrom(java.lang.String)", - "public abstract java.util.Optional lastRequestFrom(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.config.provision.HostName": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.Comparable" - ], - "attributes": [ - "public" - ], - "methods": [ - "public java.lang.String value()", - "public static com.yahoo.config.provision.HostName from(java.lang.String)", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public java.lang.String toString()", - "public int compareTo(com.yahoo.config.provision.HostName)", - "public bridge synthetic int compareTo(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.config.provision.HostSpec": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.Comparable" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, java.util.Optional)", - "public void <init>(java.lang.String, com.yahoo.config.provision.ClusterMembership, com.yahoo.config.provision.Flavor, java.util.Optional)", - "public void <init>(java.lang.String, java.util.List)", - "public void <init>(java.lang.String, java.util.List, com.yahoo.config.provision.Flavor)", - "public void <init>(java.lang.String, java.util.List, com.yahoo.config.provision.ClusterMembership)", - "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional)", - "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional)", - "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional)", - "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional)", - "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional)", - "public java.lang.String hostname()", - "public java.util.List aliases()", - "public java.util.Optional flavor()", - "public java.util.Optional version()", - "public java.util.Optional membership()", - "public java.util.Optional networkPorts()", - "public java.util.Optional requestedResources()", - "public java.util.Optional dockerImageRepo()", - "public com.yahoo.config.provision.HostSpec withPorts(java.util.Optional)", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public int compareTo(com.yahoo.config.provision.HostSpec)", - "public bridge synthetic int compareTo(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.config.provision.InfraDeployer": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract java.util.Optional getDeployment(com.yahoo.config.provision.ApplicationId)", - "public abstract void activateAllSupportedInfraApplications(boolean)" - ], - "fields": [] - }, - "com.yahoo.config.provision.InstanceName": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.Comparable" - ], - "attributes": [ - "public" - ], - "methods": [ - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public java.lang.String toString()", - "public static com.yahoo.config.provision.InstanceName from(java.lang.String)", - "public static com.yahoo.config.provision.InstanceName defaultName()", - "public boolean isDefault()", - "public boolean isTester()", - "public java.lang.String value()", - "public int compareTo(com.yahoo.config.provision.InstanceName)", - "public bridge synthetic int compareTo(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.config.provision.NetworkPorts$Allocation": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(int, java.lang.String, java.lang.String, java.lang.String)", - "public java.lang.String key()", - "public java.lang.String toString()", - "public int hashCode()", - "public boolean equals(java.lang.Object)" - ], - "fields": [ - "public final int port", - "public final java.lang.String serviceType", - "public final java.lang.String configId", - "public final java.lang.String portSuffix" - ] - }, - "com.yahoo.config.provision.NetworkPorts": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.util.Collection)", - "public java.util.Collection allocations()", - "public int size()", - "public int hashCode()", - "public boolean equals(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.config.provision.NodeFlavors": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.config.provisioning.FlavorsConfig)", - "public java.util.List getFlavors()", - "public java.util.Optional getFlavor(java.lang.String)", - "public com.yahoo.config.provision.Flavor getFlavorOrThrow(java.lang.String)", - "public boolean exists(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.config.provision.NodeResources$DiskSpeed": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.provision.NodeResources$DiskSpeed[] values()", - "public static com.yahoo.config.provision.NodeResources$DiskSpeed valueOf(java.lang.String)", - "public static int compare(com.yahoo.config.provision.NodeResources$DiskSpeed, com.yahoo.config.provision.NodeResources$DiskSpeed)", - "public boolean compatibleWith(com.yahoo.config.provision.NodeResources$DiskSpeed)", - "public boolean isDefault()", - "public static com.yahoo.config.provision.NodeResources$DiskSpeed getDefault()" - ], - "fields": [ - "public static final enum com.yahoo.config.provision.NodeResources$DiskSpeed fast", - "public static final enum com.yahoo.config.provision.NodeResources$DiskSpeed slow", - "public static final enum com.yahoo.config.provision.NodeResources$DiskSpeed any" - ] - }, - "com.yahoo.config.provision.NodeResources$StorageType": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.provision.NodeResources$StorageType[] values()", - "public static com.yahoo.config.provision.NodeResources$StorageType valueOf(java.lang.String)", - "public static int compare(com.yahoo.config.provision.NodeResources$StorageType, com.yahoo.config.provision.NodeResources$StorageType)", - "public boolean compatibleWith(com.yahoo.config.provision.NodeResources$StorageType)", - "public boolean isDefault()", - "public static com.yahoo.config.provision.NodeResources$StorageType getDefault()" - ], - "fields": [ - "public static final enum com.yahoo.config.provision.NodeResources$StorageType remote", - "public static final enum com.yahoo.config.provision.NodeResources$StorageType local", - "public static final enum com.yahoo.config.provision.NodeResources$StorageType any" - ] - }, - "com.yahoo.config.provision.NodeResources": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(double, double, double, double)", - "public void <init>(double, double, double, double, com.yahoo.config.provision.NodeResources$DiskSpeed)", - "public void <init>(double, double, double, double, com.yahoo.config.provision.NodeResources$DiskSpeed, com.yahoo.config.provision.NodeResources$StorageType)", - "public double vcpu()", - "public double memoryGb()", - "public double diskGb()", - "public double bandwidthGbps()", - "public com.yahoo.config.provision.NodeResources$DiskSpeed diskSpeed()", - "public com.yahoo.config.provision.NodeResources$StorageType storageType()", - "public com.yahoo.config.provision.NodeResources withVcpu(double)", - "public com.yahoo.config.provision.NodeResources withMemoryGb(double)", - "public com.yahoo.config.provision.NodeResources withDiskGb(double)", - "public com.yahoo.config.provision.NodeResources withBandwidthGbps(double)", - "public com.yahoo.config.provision.NodeResources with(com.yahoo.config.provision.NodeResources$DiskSpeed)", - "public com.yahoo.config.provision.NodeResources with(com.yahoo.config.provision.NodeResources$StorageType)", - "public com.yahoo.config.provision.NodeResources justNumbers()", - "public com.yahoo.config.provision.NodeResources subtract(com.yahoo.config.provision.NodeResources)", - "public com.yahoo.config.provision.NodeResources add(com.yahoo.config.provision.NodeResources)", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()", - "public boolean satisfies(com.yahoo.config.provision.NodeResources)", - "public boolean compatibleWith(com.yahoo.config.provision.NodeResources)", - "public static com.yahoo.config.provision.NodeResources fromLegacyName(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.config.provision.NodeType": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.provision.NodeType[] values()", - "public static com.yahoo.config.provision.NodeType valueOf(java.lang.String)", - "public boolean isDockerHost()", - "public java.lang.String description()", - "public com.yahoo.config.provision.NodeType childNodeType()", - "public java.util.List childNodeTypes()", - "public boolean canRun(com.yahoo.config.provision.NodeType)" - ], - "fields": [ - "public static final enum com.yahoo.config.provision.NodeType tenant", - "public static final enum com.yahoo.config.provision.NodeType host", - "public static final enum com.yahoo.config.provision.NodeType proxy", - "public static final enum com.yahoo.config.provision.NodeType proxyhost", - "public static final enum com.yahoo.config.provision.NodeType config", - "public static final enum com.yahoo.config.provision.NodeType confighost", - "public static final enum com.yahoo.config.provision.NodeType controller", - "public static final enum com.yahoo.config.provision.NodeType controllerhost", - "public static final enum com.yahoo.config.provision.NodeType devhost" - ] - }, - "com.yahoo.config.provision.OutOfCapacityException": { - "superClass": "java.lang.RuntimeException", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.config.provision.ParentHostUnavailableException": { - "superClass": "com.yahoo.config.provision.TransientException", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.config.provision.ProvisionLogger": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void log(java.util.logging.Level, java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.config.provision.Provisioner": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract java.util.List prepare(com.yahoo.config.provision.ApplicationId, com.yahoo.config.provision.ClusterSpec, com.yahoo.config.provision.Capacity, int, com.yahoo.config.provision.ProvisionLogger)", - "public abstract void activate(com.yahoo.transaction.NestedTransaction, com.yahoo.config.provision.ApplicationId, java.util.Collection)", - "public abstract void remove(com.yahoo.transaction.NestedTransaction, com.yahoo.config.provision.ApplicationId)", - "public abstract void restart(com.yahoo.config.provision.ApplicationId, com.yahoo.config.provision.HostFilter)" - ], - "fields": [] - }, - "com.yahoo.config.provision.RegionName": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.Comparable" - ], - "attributes": [ - "public" - ], - "methods": [ - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public java.lang.String toString()", - "public static com.yahoo.config.provision.RegionName from(java.lang.String)", - "public static com.yahoo.config.provision.RegionName defaultName()", - "public boolean isDefault()", - "public java.lang.String value()", - "public int compareTo(com.yahoo.config.provision.RegionName)", - "public bridge synthetic int compareTo(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.config.provision.SystemName": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.provision.SystemName[] values()", - "public static com.yahoo.config.provision.SystemName valueOf(java.lang.String)", - "public static com.yahoo.config.provision.SystemName defaultSystem()", - "public static com.yahoo.config.provision.SystemName from(java.lang.String)", - "public java.lang.String value()", - "public boolean isPublic()", - "public boolean isCd()", - "public static java.util.Set all()", - "public static java.util.Set allOf(java.util.function.Predicate)" - ], - "fields": [ - "public static final enum com.yahoo.config.provision.SystemName cd", - "public static final enum com.yahoo.config.provision.SystemName main", - "public static final enum com.yahoo.config.provision.SystemName Public", - "public static final enum com.yahoo.config.provision.SystemName PublicCd", - "public static final enum com.yahoo.config.provision.SystemName dev" - ] - }, - "com.yahoo.config.provision.TenantName": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.Comparable" - ], - "attributes": [ - "public" - ], - "methods": [ - "public java.lang.String value()", - "public static com.yahoo.config.provision.TenantName from(java.lang.String)", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public java.lang.String toString()", - "public static com.yahoo.config.provision.TenantName defaultName()", - "public int compareTo(com.yahoo.config.provision.TenantName)", - "public bridge synthetic int compareTo(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.config.provision.TransientException": { - "superClass": "java.lang.RuntimeException", - "interfaces": [], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "public void <init>(java.lang.String)", - "public void <init>(java.lang.String, java.lang.Throwable)" - ], - "fields": [] - }, - "com.yahoo.config.provision.Zone": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.cloud.config.ConfigserverConfig, com.yahoo.config.provision.NodeFlavors)", - "public void <init>(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", - "public void <init>(com.yahoo.config.provision.SystemName, com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", - "public void <init>(com.yahoo.config.provision.CloudName, com.yahoo.config.provision.SystemName, com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", - "public com.yahoo.config.provision.CloudName cloud()", - "public com.yahoo.config.provision.SystemName system()", - "public com.yahoo.config.provision.Environment environment()", - "public com.yahoo.config.provision.RegionName region()", - "public java.util.Optional nodeFlavors()", - "public static com.yahoo.config.provision.Zone defaultZone()", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - } -}
\ No newline at end of file +{}
\ No newline at end of file diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java b/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java index 208640033fc..c85c00d8ebe 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi package com.yahoo.config.provision; import com.yahoo.api.annotations.PublicApi; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index 7e9da53b1c9..0cb4baab790 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.systemflags.v1; +import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.json.FlagData; @@ -13,11 +14,13 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -47,7 +50,7 @@ public class SystemFlagsDataArchive { ZipEntry entry; while ((entry = zipIn.getNextEntry()) != null) { String name = entry.getName(); - if (!entry.isDirectory() && name.startsWith("flags/") && name.endsWith(".json")) { + if (!entry.isDirectory() && name.startsWith("flags/")) { Path filePath = Paths.get(name); String rawData = new String(zipIn.readAllBytes(), StandardCharsets.UTF_8); addFile(builder, rawData, filePath); @@ -70,7 +73,7 @@ public class SystemFlagsDataArchive { directoryStream.forEach(absolutePath -> { Path relativePath = root.relativize(absolutePath); if (!Files.isDirectory(absolutePath) && - relativePath.startsWith("flags") && relativePath.toString().endsWith(".json")) { + relativePath.startsWith("flags")) { String rawData = uncheck(() -> Files.readString(absolutePath, StandardCharsets.UTF_8)); addFile(builder, rawData, relativePath); } @@ -86,7 +89,7 @@ public class SystemFlagsDataArchive { files.forEach((flagId, fileMap) -> { fileMap.forEach((filename, flagData) -> { uncheck(() -> { - zipOut.putNextEntry(new ZipEntry("flags/" + flagId.toString() + "/" + filename)); + zipOut.putNextEntry(new ZipEntry(toFilePath(flagId, filename))); zipOut.write(flagData.serializeToUtf8Json()); zipOut.closeEntry(); }); @@ -112,8 +115,33 @@ public class SystemFlagsDataArchive { return targetData; } + public void validateAllFilesAreForTargets(SystemName currentSystem, Set<FlagsTarget> targets) throws IllegalArgumentException { + Set<String> validFiles = targets.stream() + .flatMap(target -> target.flagDataFilesPrioritized().stream()) + .collect(Collectors.toSet()); + Set<SystemName> otherSystems = Arrays.stream(SystemName.values()) + .filter(systemName -> systemName != currentSystem) + .collect(Collectors.toSet()); + files.forEach((flagId, fileMap) -> { + for (String filename : fileMap.keySet()) { + boolean isFileForOtherSystem = otherSystems.stream() + .anyMatch(system -> filename.startsWith(system.value() + ".")); + boolean isFileForCurrentSystem = validFiles.contains(filename); + if (!isFileForOtherSystem && !isFileForCurrentSystem) { + throw new IllegalArgumentException("Unknown flag file: " + toFilePath(flagId, filename)); + } + } + }); + } + private static void addFile(Builder builder, String rawData, Path filePath) { String filename = filePath.getFileName().toString(); + if (filename.startsWith(".")) { + return; // Ignore files starting with '.' + } + if (!filename.endsWith(".json")) { + throw new IllegalArgumentException(String.format("Only JSON files are allowed in 'flags/' directory (found '%s')", filePath.toString())); + } FlagId directoryDeducedFlagId = new FlagId(filePath.getName(1).toString()); FlagData flagData; if (rawData.isBlank()) { @@ -129,6 +157,10 @@ public class SystemFlagsDataArchive { builder.addFile(filename, flagData); } + private static String toFilePath(FlagId flagId, String filename) { + return "flags/" + flagId.toString() + "/" + filename; + } + public static class Builder { private final Map<FlagId, Map<String, FlagData>> files = new TreeMap<>(); diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index 14584650d3b..6e78b8da91c 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.flags.RawFlag; import com.yahoo.vespa.flags.json.FlagData; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import java.io.BufferedInputStream; @@ -27,6 +28,7 @@ import java.net.URI; import java.nio.file.Paths; import java.util.List; import java.util.Map; +import java.util.Set; import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; @@ -43,6 +45,9 @@ public class SystemFlagsDataArchiveTest { @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule + public final ExpectedException expectedException = ExpectedException.none(); + private static FlagsTarget mainControllerTarget = FlagsTarget.forController(SYSTEM); private static FlagsTarget prodUsWestCfgTarget = createConfigserverTarget(Environment.prod, "us-west-1"); private static FlagsTarget prodUsEast3CfgTarget = createConfigserverTarget(Environment.prod, "us-east-3"); @@ -84,6 +89,21 @@ public class SystemFlagsDataArchiveTest { assertFlagDataHasValue(archive, FLAG_WITH_EMPTY_DATA, devUsEast1CfgTarget, "main"); } + @Test + public void throws_exception_on_non_json_file() { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Only JSON files are allowed in 'flags/' directory (found 'flags/my-test-flag/file-name-without-dot-json')"); + SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-with-invalid-file-name/")); + } + + @Test + public void throws_exception_on_unknown_file() { + SystemFlagsDataArchive archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-with-unknown-file-name/")); + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Unknown flag file: flags/my-test-flag/main.prod.unknown-region.json"); + archive.validateAllFilesAreForTargets(SystemName.main, Set.of(mainControllerTarget, prodUsWestCfgTarget)); + } + private static void assertArchiveReturnsCorrectTestFlagDataForTarget(SystemFlagsDataArchive archive) { assertFlagDataHasValue(archive, MY_TEST_FLAG, mainControllerTarget, "main.controller"); assertFlagDataHasValue(archive, MY_TEST_FLAG, prodUsWestCfgTarget, "main.prod.us-west-1"); diff --git a/controller-api/src/test/resources/system-flags-with-invalid-file-name/flags/my-test-flag/file-name-without-dot-json b/controller-api/src/test/resources/system-flags-with-invalid-file-name/flags/my-test-flag/file-name-without-dot-json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-with-invalid-file-name/flags/my-test-flag/file-name-without-dot-json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +}
\ No newline at end of file diff --git a/controller-api/src/test/resources/system-flags-with-unknown-file-name/flags/my-test-flag/main.prod.unknown-region.json b/controller-api/src/test/resources/system-flags-with-unknown-file-name/flags/my-test-flag/main.prod.unknown-region.json new file mode 100644 index 00000000000..5924eb860c0 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-with-unknown-file-name/flags/my-test-flag/main.prod.unknown-region.json @@ -0,0 +1,8 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "value" : "default" + } + ] +}
\ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java index bc00090dc88..c55d9d9a5e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java @@ -218,6 +218,10 @@ class SystemFlagsDeployResult { return new OperationError(message, Set.of(target), OperationType.DELETE, id, null); } + static OperationError archiveValidationFailed(String message) { + return new OperationError(message, Set.of(), OperationType.VALIDATE_ARCHIVE, null, null); + } + String message() { return message; } Set<FlagsTarget> targets() { return targets; } OperationType operation() { return operation; } @@ -251,7 +255,7 @@ class SystemFlagsDeployResult { } enum OperationType { - CREATE("create"), DELETE("delete"), UPDATE("update"), LIST("list"); + CREATE("create"), DELETE("delete"), UPDATE("update"), LIST("list"), VALIDATE_ARCHIVE("validate-archive"); private final String stringValue; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java index b89a9dd7f5a..c75a6d7b413 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.systemflags; import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.config.provision.SystemName; import com.yahoo.log.LogLevel; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.flags.FlagId; @@ -38,16 +39,18 @@ class SystemFlagsDeployer { private static final Logger log = Logger.getLogger(SystemFlagsDeployer.class.getName()); private final FlagsClient client; + private final SystemName system; private final Set<FlagsTarget> targets; private final ExecutorService executor = Executors.newCachedThreadPool(new DaemonThreadFactory("system-flags-deployer-")); - SystemFlagsDeployer(ServiceIdentityProvider identityProvider, Set<FlagsTarget> targets) { - this(new FlagsClient(identityProvider, targets), targets); + SystemFlagsDeployer(ServiceIdentityProvider identityProvider, SystemName system, Set<FlagsTarget> targets) { + this(new FlagsClient(identityProvider, targets), system, targets); } - SystemFlagsDeployer(FlagsClient client, Set<FlagsTarget> targets) { + SystemFlagsDeployer(FlagsClient client, SystemName system, Set<FlagsTarget> targets) { this.client = client; + this.system = system; this.targets = targets; } @@ -65,6 +68,11 @@ class SystemFlagsDeployer { throw new RuntimeException(e); } }); + try { + archive.validateAllFilesAreForTargets(system, targets); + } catch (IllegalArgumentException e) { + results.add(new SystemFlagsDeployResult(List.of(OperationError.archiveValidationFailed(e.getMessage())))); + } return SystemFlagsDeployResult.merge(results); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java index fd594cd57bc..2f7716aaff0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java @@ -35,7 +35,7 @@ public class SystemFlagsHandler extends LoggingRequestHandler { Executor executor, AccessLog accessLog) { super(executor, accessLog); - this.deployer = new SystemFlagsDeployer(identityProvider, FlagsTarget.getAllTargetsInSystem(zoneRegistry)); + this.deployer = new SystemFlagsDeployer(identityProvider, zoneRegistry.system(), FlagsTarget.getAllTargetsInSystem(zoneRegistry)); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java index 1eb4523c344..4c41b54585f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java @@ -58,7 +58,7 @@ public class SystemFlagsDeployerTest { .build(); SystemFlagsDeployer deployer = - new SystemFlagsDeployer(flagsClient, Set.of(controllerTarget, prodUsWest1Target, prodUsEast3Target)); + new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(controllerTarget, prodUsWest1Target, prodUsEast3Target)); SystemFlagsDeployResult result = deployer.deployFlags(archive, false); @@ -83,7 +83,7 @@ public class SystemFlagsDeployerTest { .addFile("main.json", defaultData) .build(); - SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, Set.of(controllerTarget)); + SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(controllerTarget)); SystemFlagsDeployResult result = deployer.deployFlags(archive, true); verify(flagsClient, times(1)).listFlagData(controllerTarget); @@ -107,7 +107,7 @@ public class SystemFlagsDeployerTest { .addFile("main.json", defaultData) .build(); - SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, Set.of(prodUsWest1Target, prodUsEast3Target)); + SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(prodUsWest1Target, prodUsEast3Target)); SystemFlagsDeployResult result = deployer.deployFlags(archive, false); @@ -117,6 +117,21 @@ public class SystemFlagsDeployerTest { FlagDataChange.created(FLAG_ID, prodUsEast3Target, defaultData)); } + @Test + public void creates_error_entry_for_invalid_flag_archive() throws IOException { + FlagsClient flagsClient = mock(FlagsClient.class); + FlagData defaultData = flagData("flags/my-flag/main.json"); + SystemFlagsDataArchive archive = new SystemFlagsDataArchive.Builder() + .addFile("main.prod.unknown-region.json", defaultData) + .build(); + SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(controllerTarget)); + SystemFlagsDeployResult result = deployer.deployFlags(archive, false); + assertThat(result.flagChanges()) + .isEmpty(); + assertThat(result.errors()) + .containsOnly(OperationError.archiveValidationFailed("Unknown flag file: flags/my-flag/main.prod.unknown-region.json")); + } + private static FlagData flagData(String filename) throws IOException { return FlagData.deserializeUtf8Json(Files.readAllBytes(Paths.get("src/test/resources/system-flags/" + filename))); } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 8a6e833bd1f..91599f8218a 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -315,6 +315,8 @@ DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s, handleSync(*_transLog); _writtenSerial = _range.to(); _syncedSerial = _writtenSerial; + assert(int64_t(byteSize()) == _transLog->GetSize()); + assert(int64_t(byteSize()) == _transLog->GetPosition()); } DomainPart::~DomainPart() @@ -402,7 +404,7 @@ DomainPart::erase(SerialNum to) void DomainPart::commit(SerialNum firstSerial, const Packet &packet) { - int64_t firstPos(_transLog->GetPosition()); + int64_t firstPos(byteSize()); nbostream_longlivedbuf h(packet.getHandle().data(), packet.getHandle().size()); if (_range.from() == 0) { _range.from(firstSerial); @@ -576,7 +578,7 @@ DomainPart::visit(FastOS_FileInterface &file, SerialNumRange &r, Packet &packet) void DomainPart::write(FastOS_FileInterface &file, const Packet::Entry &entry) { - int64_t lastKnownGoodPos(file.GetPosition()); + int64_t lastKnownGoodPos(byteSize()); int32_t crc(0); uint32_t len(entry.serializedSize() + sizeof(crc)); nbostream os; diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp index b946b788391..7e8eaaff15e 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp @@ -241,7 +241,8 @@ void ExternalOperationHandler::bounce_or_invoke_read_only_op( IMPL_MSG_COMMAND_H(ExternalOperationHandler, Put) { - auto& metrics = getMetrics().puts[cmd->getLoadType()]; + const documentapi::LoadType & loadType = cmd->getLoadType(); + auto& metrics = getMetrics().puts[loadType]; if (!checkTimestampMutationPreconditions(*cmd, getBucketId(cmd->getDocumentId()), metrics)) { return true; } @@ -252,9 +253,10 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Put) auto handle = _mutationSequencer.try_acquire(cmd->getDocumentId()); if (allowMutation(handle)) { + document::BucketSpace bucketSpace = cmd->getBucket().getBucketSpace(); _op = std::make_shared<PutOperation>(*this, - _bucketSpaceRepo.get(cmd->getBucket().getBucketSpace()), - cmd, getMetrics().puts[cmd->getLoadType()], std::move(handle)); + _bucketSpaceRepo.get(bucketSpace), + std::move(cmd), getMetrics().puts[loadType], std::move(handle)); } else { sendUp(makeConcurrentMutationRejectionReply(*cmd, cmd->getDocumentId(), metrics)); } @@ -265,7 +267,8 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Put) IMPL_MSG_COMMAND_H(ExternalOperationHandler, Update) { - auto& metrics = getMetrics().updates[cmd->getLoadType()]; + const documentapi::LoadType & loadType = cmd->getLoadType(); + auto& metrics = getMetrics().updates[loadType]; if (!checkTimestampMutationPreconditions(*cmd, getBucketId(cmd->getDocumentId()), metrics)) { return true; } @@ -275,9 +278,10 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Update) } auto handle = _mutationSequencer.try_acquire(cmd->getDocumentId()); if (allowMutation(handle)) { + document::BucketSpace bucketSpace = cmd->getBucket().getBucketSpace(); _op = std::make_shared<TwoPhaseUpdateOperation>(*this, - _bucketSpaceRepo.get(cmd->getBucket().getBucketSpace()), - cmd, getMetrics(), std::move(handle)); + _bucketSpaceRepo.get(bucketSpace), + std::move(cmd), getMetrics(), std::move(handle)); } else { sendUp(makeConcurrentMutationRejectionReply(*cmd, cmd->getDocumentId(), metrics)); } @@ -288,7 +292,8 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Update) IMPL_MSG_COMMAND_H(ExternalOperationHandler, Remove) { - auto& metrics = getMetrics().removes[cmd->getLoadType()]; + const documentapi::LoadType & loadType = cmd->getLoadType(); + auto& metrics = getMetrics().removes[loadType]; if (!checkTimestampMutationPreconditions(*cmd, getBucketId(cmd->getDocumentId()), metrics)) { return true; } @@ -299,8 +304,9 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Remove) auto handle = _mutationSequencer.try_acquire(cmd->getDocumentId()); if (allowMutation(handle)) { auto &distributorBucketSpace(_bucketSpaceRepo.get(cmd->getBucket().getBucketSpace())); - _op = std::make_shared<RemoveOperation>(*this, distributorBucketSpace, cmd, - getMetrics().removes[cmd->getLoadType()], std::move(handle)); + + _op = std::make_shared<RemoveOperation>(*this, distributorBucketSpace, std::move(cmd), + getMetrics().removes[loadType], std::move(handle)); } else { sendUp(makeConcurrentMutationRejectionReply(*cmd, cmd->getDocumentId(), metrics)); } @@ -320,7 +326,7 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, RemoveLocation) } _op = std::make_shared<RemoveLocationOperation>(*this, _bucketSpaceRepo.get(cmd->getBucket().getBucketSpace()), - cmd, getMetrics().removelocations[cmd->getLoadType()]); + std::move(cmd), getMetrics().removelocations[cmd->getLoadType()]); return true; } diff --git a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp index 71d8f1b17ea..ca1b6f266d6 100644 --- a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp @@ -18,30 +18,28 @@ using document::BucketSpace; RemoveLocationOperation::RemoveLocationOperation( DistributorComponent& manager, DistributorBucketSpace &bucketSpace, - const std::shared_ptr<api::RemoveLocationCommand> & msg, + std::shared_ptr<api::RemoveLocationCommand> msg, PersistenceOperationMetricSet& metric) : Operation(), _trackerInstance(metric, - std::shared_ptr<api::BucketInfoReply>(new api::RemoveLocationReply(*msg)), + std::make_shared<api::RemoveLocationReply>(*msg), manager, 0), _tracker(_trackerInstance), - _msg(msg), + _msg(std::move(msg)), _manager(manager), _bucketSpace(bucketSpace) {} -RemoveLocationOperation::~RemoveLocationOperation() {} +RemoveLocationOperation::~RemoveLocationOperation() = default; int RemoveLocationOperation::getBucketId( DistributorComponent& manager, const api::RemoveLocationCommand& cmd, document::BucketId& bid) { - std::shared_ptr<const document::DocumentTypeRepo> repo = - manager.getTypeRepo(); - document::select::Parser parser( - *repo, manager.getBucketIdFactory()); + std::shared_ptr<const document::DocumentTypeRepo> repo = manager.getTypeRepo(); + document::select::Parser parser(*repo, manager.getBucketIdFactory()); document::BucketSelector bucketSel(manager.getBucketIdFactory()); std::unique_ptr<document::BucketSelector::BucketVector> exprResult diff --git a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.h b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.h index 64aeb19bae9..bad55497fa1 100644 --- a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.h @@ -17,9 +17,9 @@ class RemoveLocationOperation : public Operation public: RemoveLocationOperation(DistributorComponent& manager, DistributorBucketSpace &bucketSpace, - const std::shared_ptr<api::RemoveLocationCommand> & msg, + std::shared_ptr<api::RemoveLocationCommand> msg, PersistenceOperationMetricSet& metric); - ~RemoveLocationOperation(); + ~RemoveLocationOperation() override; static int getBucketId(DistributorComponent& manager, diff --git a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp index 0ad3d282cc1..9211e75ca80 100644 --- a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp @@ -13,21 +13,21 @@ using document::BucketSpace; RemoveOperation::RemoveOperation(DistributorComponent& manager, DistributorBucketSpace &bucketSpace, - const std::shared_ptr<api::RemoveCommand> & msg, + std::shared_ptr<api::RemoveCommand> msg, PersistenceOperationMetricSet& metric, SequencingHandle sequencingHandle) : SequencedOperation(std::move(sequencingHandle)), _trackerInstance(metric, - std::shared_ptr<api::BucketInfoReply>(new api::RemoveReply(*msg)), + std::make_shared<api::RemoveReply>(*msg), manager, msg->getTimestamp()), _tracker(_trackerInstance), - _msg(msg), + _msg(std::move(msg)), _manager(manager), _bucketSpace(bucketSpace) { } -RemoveOperation::~RemoveOperation() {} +RemoveOperation::~RemoveOperation() = default; void RemoveOperation::onStart(DistributorMessageSender& sender) @@ -43,22 +43,19 @@ RemoveOperation::onStart(DistributorMessageSender& sender) bool sent = false; - for (uint32_t j = 0; j < entries.size(); j++) { - const BucketDatabase::Entry& e = entries[j]; + for (const BucketDatabase::Entry& e : entries) { std::vector<MessageTracker::ToSend> messages; - + messages.reserve(e->getNodeCount()); for (uint32_t i = 0; i < e->getNodeCount(); i++) { - std::shared_ptr<api::RemoveCommand> command(new api::RemoveCommand( - document::Bucket(_msg->getBucket().getBucketSpace(), e.getBucketId()), - _msg->getDocumentId(), - _msg->getTimestamp())); + auto command = std::make_shared<api::RemoveCommand>(document::Bucket(_msg->getBucket().getBucketSpace(), e.getBucketId()), + _msg->getDocumentId(), + _msg->getTimestamp()); copyMessageSettings(*_msg, *command); command->getTrace().setLevel(_msg->getTrace().getLevel()); command->setCondition(_msg->getCondition()); - messages.push_back( - MessageTracker::ToSend(command, e->getNodeRef(i).getNode())); + messages.emplace_back(std::move(command), e->getNodeRef(i).getNode()); sent = true; } diff --git a/storage/src/vespa/storage/distributor/operations/external/removeoperation.h b/storage/src/vespa/storage/distributor/operations/external/removeoperation.h index 7794be73ac8..90e7ac94f0a 100644 --- a/storage/src/vespa/storage/distributor/operations/external/removeoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/removeoperation.h @@ -17,10 +17,10 @@ class RemoveOperation : public SequencedOperation public: RemoveOperation(DistributorComponent& manager, DistributorBucketSpace &bucketSpace, - const std::shared_ptr<api::RemoveCommand> & msg, + std::shared_ptr<api::RemoveCommand> msg, PersistenceOperationMetricSet& metric, SequencingHandle sequencingHandle = SequencingHandle()); - ~RemoveOperation(); + ~RemoveOperation() override; void onStart(DistributorMessageSender& sender) override; const char* getName() const override { return "remove"; }; diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index dd8f8b553fc..706b7a447bf 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -22,14 +22,14 @@ namespace storage::distributor { TwoPhaseUpdateOperation::TwoPhaseUpdateOperation( DistributorComponent& manager, DistributorBucketSpace &bucketSpace, - const std::shared_ptr<api::UpdateCommand>& msg, + std::shared_ptr<api::UpdateCommand> msg, DistributorMetricSet& metrics, SequencingHandle sequencingHandle) : SequencedOperation(std::move(sequencingHandle)), _updateMetric(metrics.updates[msg->getLoadType()]), _putMetric(metrics.update_puts[msg->getLoadType()]), _getMetric(metrics.update_gets[msg->getLoadType()]), - _updateCmd(msg), + _updateCmd(std::move(msg)), _updateReply(), _manager(manager), _bucketSpace(bucketSpace), diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h index f714232b427..4a2f83010c7 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h @@ -56,7 +56,7 @@ class TwoPhaseUpdateOperation : public SequencedOperation public: TwoPhaseUpdateOperation(DistributorComponent& manager, DistributorBucketSpace &bucketSpace, - const std::shared_ptr<api::UpdateCommand> & msg, + std::shared_ptr<api::UpdateCommand> msg, DistributorMetricSet& metrics, SequencingHandle sequencingHandle = SequencingHandle()); ~TwoPhaseUpdateOperation() override; diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp b/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp index 060eda2ffc5..4e2253ab2fe 100644 --- a/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp +++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp @@ -20,33 +20,26 @@ PersistenceMessageTrackerImpl::PersistenceMessageTrackerImpl( api::Timestamp revertTimestamp) : MessageTracker(link.getClusterName()), _metric(metric), - _reply(reply), + _reply(std::move(reply)), _manager(link), _revertTimestamp(revertTimestamp), _requestTimer(link.getClock()), - _priority(reply->getPriority()), + _priority(_reply->getPriority()), _success(true) { } -PersistenceMessageTrackerImpl::~PersistenceMessageTrackerImpl() {} +PersistenceMessageTrackerImpl::~PersistenceMessageTrackerImpl() = default; void PersistenceMessageTrackerImpl::updateDB() { - for (BucketInfoMap::iterator iter = _bucketInfo.begin(); - iter != _bucketInfo.end(); - iter++) - { - _manager.updateBucketDatabase(iter->first, iter->second); + for (const auto & entry : _bucketInfo) { + _manager.updateBucketDatabase(entry.first, entry.second); } - for (BucketInfoMap::iterator iter = _remapBucketInfo.begin(); - iter != _remapBucketInfo.end(); - iter++) - { - _manager.updateBucketDatabase(iter->first, iter->second, - DatabaseUpdate::CREATE_IF_NONEXISTING); + for (const auto & entry : _remapBucketInfo){ + _manager.updateBucketDatabase(entry.first, entry.second,DatabaseUpdate::CREATE_IF_NONEXISTING); } } @@ -94,11 +87,10 @@ PersistenceMessageTrackerImpl::revert( std::vector<api::Timestamp> reverts; reverts.push_back(_revertTimestamp); - for (uint32_t i = 0; i < revertNodes.size(); i++) { - std::shared_ptr<api::RevertCommand> toRevert( - new api::RevertCommand(revertNodes[i].first, reverts)); + for (const auto & revertNode : revertNodes) { + auto toRevert = std::make_shared<api::RevertCommand>(revertNode.first, reverts); toRevert->setPriority(_priority); - queueCommand(toRevert, revertNodes[i].second); + queueCommand(std::move(toRevert), revertNode.second); } flushQueue(sender); @@ -107,14 +99,14 @@ PersistenceMessageTrackerImpl::revert( void PersistenceMessageTrackerImpl::queueMessageBatch(const std::vector<MessageTracker::ToSend>& messages) { - _messageBatches.push_back(MessageBatch()); - for (uint32_t i = 0; i < messages.size(); i++) { - if (_reply.get()) { - messages[i]._msg->getTrace().setLevel(_reply->getTrace().getLevel()); + _messageBatches.emplace_back(); + for (const auto & message : messages) { + if (_reply) { + message._msg->getTrace().setLevel(_reply->getTrace().getLevel()); } - _messageBatches.back().push_back(messages[i]._msg->getMsgId()); - queueCommand(messages[i]._msg, messages[i]._target); + _messageBatches.back().push_back(message._msg->getMsgId()); + queueCommand(message._msg, message._target); } } @@ -134,13 +126,13 @@ PersistenceMessageTrackerImpl::canSendReplyEarly() const return false; } - for (uint32_t i = 0; i < _messageBatches.size(); i++) { + for (const MessageBatch & batch : _messageBatches) { uint32_t messagesDone = 0; - for (uint32_t j = 0; j < _messageBatches[i].size(); j++) { - if (_sentMessages.find(_messageBatches[i][j]) == _sentMessages.end()) { + for (uint32_t i = 0; i < batch.size(); i++) { + if (_sentMessages.find(batch[i]) == _sentMessages.end()) { messagesDone++; - } else if (distribution.ensurePrimaryPersisted() && j == 0) { + } else if (distribution.ensurePrimaryPersisted() && i == 0) { // Primary must always be written. LOG(debug, "Not returning early because primary node wasn't done"); return false; @@ -160,20 +152,17 @@ PersistenceMessageTrackerImpl::canSendReplyEarly() const void PersistenceMessageTrackerImpl::checkCopiesDeleted() { - if (!_reply.get()) { + if ( ! _reply) { return; } // Don't check the buckets that have been remapped here, as we will // create them. const auto &bucketSpaceRepo(_manager.getBucketSpaceRepo()); - for (BucketInfoMap::const_iterator iter = _bucketInfo.begin(); - iter != _bucketInfo.end(); - iter++) - { - const auto &bucketSpace(bucketSpaceRepo.get(iter->first.getBucketSpace())); + for (const auto & entry : _bucketInfo) { + const auto &bucketSpace(bucketSpaceRepo.get(entry.first.getBucketSpace())); const auto &bucketDb(bucketSpace.getBucketDatabase()); - BucketDatabase::Entry dbentry = bucketDb.get(iter->first.getBucketId()); + BucketDatabase::Entry dbentry = bucketDb.get(entry.first.getBucketId()); if (!dbentry.valid()) { continue; @@ -182,17 +171,17 @@ PersistenceMessageTrackerImpl::checkCopiesDeleted() std::vector<uint16_t> missing; std::vector<uint16_t> total; - for (uint32_t i = 0; i < iter->second.size(); ++i) { - if (dbentry->getNode(iter->second[i].getNode()) == NULL) { - missing.push_back(iter->second[i].getNode()); + for (const BucketCopy & bucketCopy : entry.second) { + if (dbentry->getNode(bucketCopy.getNode()) == nullptr) { + missing.push_back(bucketCopy.getNode()); } - total.push_back(iter->second[i].getNode()); + total.push_back(bucketCopy.getNode()); } if (!missing.empty()) { std::ostringstream msg; - msg << iter->first.toString() << " was deleted from nodes [" + msg << entry.first.toString() << " was deleted from nodes [" << commaSeparated(missing) << "] after message was sent but before it was done. Sent to [" << commaSeparated(total) @@ -258,7 +247,7 @@ bool PersistenceMessageTrackerImpl::shouldRevert() const { return _manager.getDistributorConfig().enableRevert - && _revertNodes.size() && !_success && _reply.get(); + && !_revertNodes.empty() && !_success && _reply; } void diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.h b/storage/src/vespa/storage/distributor/persistencemessagetracker.h index 2b4e7d8edc1..5a4d8e17452 100644 --- a/storage/src/vespa/storage/distributor/persistencemessagetracker.h +++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.h @@ -90,7 +90,7 @@ private: void handlePersistenceReply(api::BucketInfoReply& reply, uint16_t node); void queueCommand(std::shared_ptr<api::BucketCommand> msg, uint16_t target) override { - MessageTracker::queueCommand(msg, target); + MessageTracker::queueCommand(std::move(msg), target); } void flushQueue(MessageSender& s) override { MessageTracker::flushQueue(s); } uint16_t handleReply(api::BucketReply& r) override { return MessageTracker::handleReply(r); } |