diff options
49 files changed, 998 insertions, 360 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index 1fd63beab39..dfed0476aab 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -24,18 +24,18 @@ public class MemoryNameService implements NameService { @Override public Record createCname(RecordName name, RecordData canonicalName) { - Record record = new Record(Record.Type.CNAME, name, canonicalName); + var record = new Record(Record.Type.CNAME, name, canonicalName); records.add(record); return record; } @Override public List<Record> createAlias(RecordName name, Set<AliasTarget> targets) { - List<Record> records = targets.stream() - .sorted((a, b) -> Comparator.comparing(AliasTarget::name).compare(a, b)) - .map(target -> new Record(Record.Type.ALIAS, name, - RecordData.fqdn(target.name().value()))) - .collect(Collectors.toList()); + var records = targets.stream() + .sorted((a, b) -> Comparator.comparing(AliasTarget::name).compare(a, b)) + .map(target -> new Record(Record.Type.ALIAS, name, + RecordData.fqdn(target.name().value()))) + .collect(Collectors.toList()); // Satisfy idempotency contract of interface removeRecords(findRecords(Record.Type.ALIAS, name)); this.records.addAll(records); @@ -44,9 +44,9 @@ public class MemoryNameService implements NameService { @Override public List<Record> createTxtRecords(RecordName name, List<RecordData> txtData) { - List<Record> records = txtData.stream() - .map(data -> new Record(Record.Type.TXT, name, data)) - .collect(Collectors.toList()); + var records = txtData.stream() + .map(data -> new Record(Record.Type.TXT, name, data)) + .collect(Collectors.toList()); this.records.addAll(records); return records; } @@ -67,7 +67,7 @@ public class MemoryNameService implements NameService { @Override public void updateRecord(Record record, RecordData newData) { - List<Record> records = findRecords(record.type(), record.name()); + var records = findRecords(record.type(), record.name()); if (records.isEmpty()) { throw new IllegalArgumentException("No record with data '" + newData.asString() + "' exists"); } @@ -75,7 +75,7 @@ public class MemoryNameService implements NameService { throw new IllegalArgumentException("Cannot update multi-value record '" + record.name().asString() + "' with '" + newData.asString() + "'"); } - Record existing = records.get(0); + var existing = records.get(0); this.records.remove(existing); this.records.add(new Record(existing.type(), existing.name(), newData)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 14a5d3c7ddf..ce7af03aa7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -26,6 +26,7 @@ public class Endpoint { private final Scope scope; private final boolean legacy; private final boolean directRouting; + private final boolean tls; private Endpoint(String name, ApplicationId application, ZoneId zone, SystemName system, Port port, boolean legacy, boolean directRouting) { @@ -37,6 +38,7 @@ public class Endpoint { this.scope = zone == null ? Scope.global : Scope.zone; this.legacy = legacy; this.directRouting = directRouting; + this.tls = port.tls; } /** Returns the URL used to access this */ @@ -67,6 +69,11 @@ public class Endpoint { return directRouting; } + /** Returns whether this endpoint supports TLS connections */ + public boolean tls() { + return tls; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java index c4b69ce5588..2fc852d79d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java @@ -23,28 +23,23 @@ import java.util.Set; public class RoutingPolicy { private final ApplicationId owner; + private final ClusterSpec.Id cluster; private final ZoneId zone; - private final HostName alias; private final HostName canonicalName; private final Optional<String> dnsZone; private final Set<RotationName> rotations; /** DO NOT USE. Public for serialization purposes */ - public RoutingPolicy(ApplicationId owner, ZoneId zone, HostName alias, HostName canonicalName, + public RoutingPolicy(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone, HostName canonicalName, Optional<String> dnsZone, Set<RotationName> rotations) { this.owner = Objects.requireNonNull(owner, "owner must be non-null"); + this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); this.zone = Objects.requireNonNull(zone, "zone must be non-null"); - this.alias = Objects.requireNonNull(alias, "alias must be non-null"); this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); this.rotations = ImmutableSortedSet.copyOf(Objects.requireNonNull(rotations, "rotations must be non-null")); } - public RoutingPolicy(ApplicationId owner, ZoneId zone, ClusterSpec.Id cluster, SystemName system, HostName canonicalName, - Optional<String> dnsZone, Set<RotationName> rotations) { - this(owner, zone, HostName.from(endpointOf(cluster, owner, zone, system).dnsName()), canonicalName, dnsZone, rotations); - } - /** The application owning this */ public ApplicationId owner() { return owner; @@ -55,9 +50,9 @@ public class RoutingPolicy { return zone; } - /** This alias (lhs of a CNAME or ALIAS record) */ - public HostName alias() { - return alias; + /** The cluster this applies to */ + public ClusterSpec.Id cluster() { + return cluster; } /** The canonical name for this (rhs of a CNAME or ALIAS record) */ @@ -75,8 +70,13 @@ public class RoutingPolicy { return rotations; } - /** Endpoints for this routing policy */ - public EndpointList endpointsIn(SystemName system) { + /** Returns the endpoint of this */ + public Endpoint endpointIn(SystemName system) { + return Endpoint.of(owner).target(cluster, zone).on(Port.tls()).directRouting().in(system); + } + + /** Returns rotation endpoints of this */ + public EndpointList rotationEndpointsIn(SystemName system) { return EndpointList.of(rotations.stream().map(rotation -> endpointOf(owner, rotation, system))); } @@ -86,19 +86,21 @@ public class RoutingPolicy { if (o == null || getClass() != o.getClass()) return false; RoutingPolicy policy = (RoutingPolicy) o; return owner.equals(policy.owner) && + cluster.equals(policy.cluster) && zone.equals(policy.zone) && - canonicalName.equals(policy.canonicalName); + canonicalName.equals(policy.canonicalName) && + dnsZone.equals(policy.dnsZone); } @Override public int hashCode() { - return Objects.hash(owner, zone, canonicalName); + return Objects.hash(owner, cluster, zone, canonicalName, dnsZone); } @Override public String toString() { - return String.format("%s -> %s [rotations: %s%s], owned by %s, in %s", alias, canonicalName, rotations, - dnsZone.map(z -> ", DNS zone: " + z).orElse(""), owner.toShortString(), + return String.format("%s [rotations: %s%s], %s owned by %s, in %s", canonicalName, rotations, + dnsZone.map(z -> ", DNS zone: " + z).orElse(""), cluster, owner.toShortString(), zone.value()); } @@ -107,9 +109,4 @@ public class RoutingPolicy { return Endpoint.of(application).target(rotation).on(Port.tls()).directRouting().in(system); } - /** Returns the endpoint of given cluster */ - public static Endpoint endpointOf(ClusterSpec.Id cluster, ApplicationId application, ZoneId zone, SystemName system) { - return Endpoint.of(application).target(cluster, zone).on(Port.tls()).directRouting().in(system); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index e6aa8bc51b5..a8130d60cc5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -98,7 +98,7 @@ public class DeploymentTrigger { report.jobType(), report.applicationId(), report.projectId())); - if ( ! applications().get(report.applicationId()).isPresent()) { + if (applications().get(report.applicationId()).isEmpty()) { log.log(LogLevel.WARNING, "Ignoring completion of job of project '" + report.projectId() + "': Unknown application '" + report.applicationId() + "'"); return; @@ -285,7 +285,7 @@ public class DeploymentTrigger { } private static <T extends Comparable<T>> Optional<T> max(Optional<T> o1, Optional<T> o2) { - return ! o1.isPresent() ? o2 : ! o2.isPresent() ? o1 : o1.get().compareTo(o2.get()) >= 0 ? o1 : o2; + return o1.isEmpty() ? o2 : o2.isEmpty() ? o1 : o1.get().compareTo(o2.get()) >= 0 ? o1 : o2; } // ---------- Ready job computation ---------- @@ -396,11 +396,11 @@ public class DeploymentTrigger { /** Returns whether the given job can trigger at the given instant */ public boolean triggerAt(Instant instant, JobType job, Versions versions, Application application) { Optional<JobStatus> jobStatus = application.deploymentJobs().statusOf(job); - if ( ! jobStatus.isPresent()) return true; + if (jobStatus.isEmpty()) return true; if (jobStatus.get().pausedUntil().isPresent() && jobStatus.get().pausedUntil().getAsLong() > clock.instant().toEpochMilli()) return false; if (jobStatus.get().isSuccess()) return true; // Success - if ( ! jobStatus.get().lastCompleted().isPresent()) return true; // Never completed - if ( ! jobStatus.get().firstFailing().isPresent()) return true; // Should not happen as firstFailing should be set for an unsuccessful job + if (jobStatus.get().lastCompleted().isEmpty()) return true; // Never completed + if (jobStatus.get().firstFailing().isEmpty()) return true; // Should not happen as firstFailing should be set for an unsuccessful job if ( ! versions.targetsMatch(jobStatus.get().lastCompleted().get())) return true; // Always trigger as targets have changed if (application.deploymentSpec().upgradePolicy() == DeploymentSpec.UpgradePolicy.canary) return true; // Don't throttle canaries @@ -516,7 +516,7 @@ public class DeploymentTrigger { if ( ! application.deploymentSpec().canChangeRevisionAt(clock.instant())) return false; if (application.change().application().isPresent()) return true; // Replacing a previous application change is ok. if (application.deploymentJobs().hasFailures()) return true; // Allow changes to fix upgrade problems. - return ! application.change().platform().isPresent(); + return application.change().platform().isEmpty(); } private Change remainingChange(Application application) { @@ -552,7 +552,7 @@ public class DeploymentTrigger { for (JobType jobType : steps(application.deploymentSpec()).testJobs()) { Optional<JobRun> completion = successOn(application, jobType, versions) .filter(run -> versions.sourcesMatchIfPresent(run) || jobType == systemTest); - if ( ! completion.isPresent() && condition.test(jobType)) + if (completion.isEmpty() && condition.test(jobType)) jobs.add(deploymentJob(application, versions, application.change(), jobType, reason, availableSince)); } return jobs; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 9133c8980ec..cd3341ed3a6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -1,16 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.yolean.Exceptions; -import java.time.Clock; import java.time.Duration; -import java.time.Instant; import java.util.logging.Level; /** @@ -21,41 +17,32 @@ import java.util.logging.Level; */ public class DeploymentExpirer extends Maintainer { - private final Clock clock; - public DeploymentExpirer(Controller controller, Duration interval, JobControl jobControl) { - this(controller, interval, Clock.systemUTC(), jobControl); - } - - public DeploymentExpirer(Controller controller, Duration interval, Clock clock, JobControl jobControl) { super(controller, interval, jobControl); - this.clock = clock; } @Override protected void maintain() { for (Application application : controller().applications().asList()) { for (Deployment deployment : application.deployments().values()) { - if (deployment.zone().environment().equals(Environment.prod)) { - continue; - } - - if (hasExpired(controller().zoneRegistry(), deployment, clock.instant())) { - try { - controller().applications().deactivate(application.id(), deployment.zone()); - } catch (Exception e) { - log.log(Level.WARNING, "Could not expire " + deployment + " of " + application + - ": " + Exceptions.toMessageString(e) + ". Retrying in " + - maintenanceInterval()); - } + if (!isExpired(deployment)) continue; + + try { + controller().applications().deactivate(application.id(), deployment.zone()); + } catch (Exception e) { + log.log(Level.WARNING, "Could not expire " + deployment + " of " + application + + ": " + Exceptions.toMessageString(e) + ". Retrying in " + + maintenanceInterval()); } } } } - private static boolean hasExpired(ZoneRegistry zoneRegistry, Deployment deployment, Instant now) { - return zoneRegistry.getDeploymentTimeToLive(deployment.zone()) - .map(timeToLive -> deployment.at().plus(timeToLive).isBefore(now)) + /** Returns whether given deployment has expired according to its TTL */ + private boolean isExpired(Deployment deployment) { + if (deployment.zone().environment().isProduction()) return false; // Never expire production deployments + return controller().zoneRegistry().getDeploymentTimeToLive(deployment.zone()) + .map(timeToLive -> deployment.at().plus(timeToLive).isBefore(controller().clock().instant())) .orElse(false); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java index 417a1944ad3..03d894f9a17 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java @@ -137,11 +137,10 @@ public class RoutingPolicyMaintainer extends Maintainer { /** Register DNS alias for given load balancer */ private RoutingPolicy registerCname(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) { - RoutingPolicy routingPolicy = new RoutingPolicy(application, zone, - loadBalancer.cluster(), controller().system(), + RoutingPolicy routingPolicy = new RoutingPolicy(application, loadBalancer.cluster(), zone, loadBalancer.hostname(), loadBalancer.dnsZone(), loadBalancer.rotations()); - RecordName name = RecordName.from(routingPolicy.alias().value()); + RecordName name = RecordName.from(routingPolicy.endpointIn(controller().system()).dnsName()); RecordData data = RecordData.fqdn(loadBalancer.hostname().value()); List<Record> existingRecords = nameService.findRecords(Record.Type.CNAME, name); if (existingRecords.size() > 1) { @@ -170,11 +169,12 @@ public class RoutingPolicyMaintainer extends Maintainer { // Remove any active load balancers removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName())); for (RoutingPolicy policy : removalCandidates) { + String dnsName = policy.endpointIn(controller().system()).dnsName(); try { - List<Record> records = nameService.findRecords(Record.Type.CNAME, RecordName.from(policy.alias().value())); + List<Record> records = nameService.findRecords(Record.Type.CNAME, RecordName.from(dnsName)); nameService.removeRecords(records); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to remove record '" + policy.alias() + + log.log(LogLevel.WARNING, "Failed to remove record '" + dnsName + "'. Retrying in " + maintenanceInterval()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 47ae8566ab8..3c2cbade606 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -47,7 +47,7 @@ import java.util.OptionalLong; import java.util.TreeMap; /** - * Serializes applications to/from slime. + * Serializes {@link Application} to/from slime. * This class is multithread safe. * * @author bratseth diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java index 93dd59e577f..36583f4320e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java @@ -14,7 +14,7 @@ import java.util.Optional; import java.util.function.Function; /** - * Slime serializer for the audit log. + * Slime serializer for {@link AuditLog}. * * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java index c56d8b3849c..a87875da104 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; - import com.yahoo.component.Version; import com.yahoo.slime.Cursor; import com.yahoo.slime.ObjectTraverser; @@ -13,7 +12,7 @@ import java.util.LinkedHashMap; import java.util.Map; /** - * Serializes overrides of version confidence. + * Serializer for {@link Confidence} overrides. * * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java index 36736f7f398..40781ac6e92 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java @@ -21,7 +21,7 @@ import java.util.Map; import java.util.stream.Collectors; /** - * Serialisation of LogRecord objects. Not all fields are stored! + * Serialisation of {@link LogEntry} objects. Not all fields are stored! * * @author jonmv */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java index d996b79fe18..21f8b1bcb80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java @@ -14,7 +14,7 @@ import java.util.Set; import java.util.TreeSet; /** - * Serializer for an OS version. + * Serializer for an {@link OsVersion}. * * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java index b0557863426..3e3c0df1673 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java @@ -20,7 +20,7 @@ import java.util.Objects; import java.util.TreeMap; /** - * Serializer for OS version status. + * Serializer for {@link OsVersionStatus}. * * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 722cde68c65..7c4f9a66fd3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RotationName; import com.yahoo.slime.ArrayTraverser; @@ -21,11 +22,12 @@ import java.util.function.Function; * Serializer and deserializer for a {@link RoutingPolicy}. * * @author mortent + * @author mpolden */ public class RoutingPolicySerializer { private static final String routingPoliciesField = "routingPolicies"; - private static final String aliasField = "alias"; + private static final String clusterField = "cluster"; private static final String canonicalNameField = "canonicalName"; private static final String zoneField = "zone"; private static final String dnsZoneField = "dnsZone"; @@ -37,7 +39,7 @@ public class RoutingPolicySerializer { Cursor policyArray = root.setArray(routingPoliciesField); routingPolicies.forEach(policy -> { Cursor policyObject = policyArray.addObject(); - policyObject.setString(aliasField, policy.alias().value()); + policyObject.setString(clusterField, policy.cluster().value()); policyObject.setString(zoneField, policy.zone().value()); policyObject.setString(canonicalNameField, policy.canonicalName().value()); policy.dnsZone().ifPresent(dnsZone -> policyObject.setString(dnsZoneField, dnsZone)); @@ -57,8 +59,8 @@ public class RoutingPolicySerializer { Set<RotationName> rotations = new LinkedHashSet<>(); inspect.field(rotationsField).traverse((ArrayTraverser) (j, rotation) -> rotations.add(RotationName.from(rotation.asString()))); policies.add(new RoutingPolicy(owner, + clusterId(inspect.field(clusterField)), ZoneId.from(inspect.field(zoneField).asString()), - HostName.from(inspect.field(aliasField).asString()), HostName.from(inspect.field(canonicalNameField).asString()), optionalField(inspect.field(dnsZoneField), Function.identity()), rotations)); @@ -66,6 +68,11 @@ public class RoutingPolicySerializer { return Collections.unmodifiableSet(policies); } + // TODO: Remove and inline after Vespa 7.43 + private static ClusterSpec.Id clusterId(Inspector field) { + return optionalField(field, ClusterSpec.Id::from).orElseGet(() -> new ClusterSpec.Id("default")); + } + private static <T> Optional<T> optionalField(Inspector field, Function<String, T> fieldMapper) { return Optional.of(field).filter(Inspector::valid).map(Inspector::asString).map(fieldMapper); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index be464f95385..ce757e015b8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -50,7 +50,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; import static java.util.Comparator.comparing; /** - * Serialises and deserialises RunStatus objects for persistent storage. + * Serialises and deserialises {@link Run} objects for persistent storage. * * @author jonmv */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 2a685914408..56e80068908 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.Optional; /** - * Slime serialization of tenants. + * Slime serialization of {@link Tenant} sub-types. * * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java index 78045a15e9c..5edae803fdb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java @@ -7,7 +7,7 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; /** - * Serializer for version numbers. + * Serializer for {@link Version}. * * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 11f0bfcfa0f..72d38bbee5f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.Set; /** - * Serializes VersionStatus to and from slime + * Serializer for {@link VersionStatus}. * * @author mpolden */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 74206d05009..1b9bf28f395 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -508,7 +508,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Per-cluster rotations Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies(application.id()); for (RoutingPolicy policy : routingPolicies) { - policy.endpointsIn(controller.system()).asList().stream() + policy.rotationEndpointsIn(controller.system()).asList().stream() .map(Endpoint::url) .map(URI::toString) .forEach(globalRotationsArray::addString); @@ -584,13 +584,22 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private void toSlime(Cursor response, DeploymentId deploymentId, Deployment deployment, HttpRequest request) { - response.setString("tenant", deploymentId.applicationId().tenant().value()); response.setString("application", deploymentId.applicationId().application().value()); response.setString("instance", deploymentId.applicationId().instance().value()); // pointless response.setString("environment", deploymentId.zoneId().environment().value()); response.setString("region", deploymentId.zoneId().region().value()); + // Add endpoint(s) defined by routing policies + var endpointArray = response.setArray("endpoints"); + for (var policy : controller.applications().routingPolicies(deploymentId.applicationId())) { + Cursor endpointObject = endpointArray.addObject(); + Endpoint endpoint = policy.endpointIn(controller.system()); + endpointObject.setString("cluster", policy.cluster().value()); + endpointObject.setBool("tls", endpoint.tls()); + endpointObject.setString("url", endpoint.url().toString()); + } + // serviceUrls contains zone/cluster-specific endpoints for this deployment. The name of these endpoints may // contain the cluster name (if non-default) and since the controller has no knowledge of clusters, we have to // ask the routing layer here diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index f642f3210b7..730b2943431 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -3,14 +3,13 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import org.junit.Before; import org.junit.Test; import java.time.Duration; @@ -24,12 +23,7 @@ import static org.junit.Assert.assertEquals; */ public class DeploymentExpirerTest { - private DeploymentTester tester; - - @Before - public void before() { - tester = new DeploymentTester(); - } + private final DeploymentTester tester = new DeploymentTester(); @Test public void testDeploymentExpiry() { @@ -38,7 +32,7 @@ public class DeploymentExpirerTest { Duration.ofDays(14) ); DeploymentExpirer expirer = new DeploymentExpirer(tester.controller(), Duration.ofDays(10), - tester.clock(), new JobControl(new MockCuratorDb())); + new JobControl(new MockCuratorDb())); Application devApp = tester.createApplication("app1", "tenant1", 123L, 1L); Application prodApp = tester.createApplication("app2", "tenant2", 456L, 2L); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java index 0541a0b05f5..b0f64eee532 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java @@ -72,7 +72,7 @@ public class RoutingPolicyMaintainerTest { assertEquals("lb-0--tenant1:app1:default--prod.us-central-1.", records2.get().get(0).data().asString()); assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records2.get().get(1).data().asString()); assertEquals(2, tester.controller().applications().routingPolicies(app1.id()).iterator().next() - .endpointsIn(SystemName.main).asList().size()); + .rotationEndpointsIn(SystemName.main).asList().size()); // Applications gains a new deployment ApplicationPackage updatedApplicationPackage = new ApplicationPackageBuilder() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index 4fe465ce01e..4a4fd39ccb7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -3,9 +3,11 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RotationName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import org.junit.Test; @@ -19,20 +21,21 @@ import static org.junit.Assert.assertEquals; */ public class RoutingPolicySerializerTest { + private final RoutingPolicySerializer serializer = new RoutingPolicySerializer(); + @Test public void test_serialization() { - RoutingPolicySerializer serializer = new RoutingPolicySerializer(); ApplicationId owner = ApplicationId.defaultId(); Set<RotationName> rotations = Set.of(RotationName.from("r1"), RotationName.from("r2")); Set<RoutingPolicy> loadBalancers = ImmutableSet.of(new RoutingPolicy(owner, + ClusterSpec.Id.from("my-cluster1"), ZoneId.from("prod", "us-north-1"), - HostName.from("my-pretty-alias"), HostName.from("long-and-ugly-name"), Optional.of("zone1"), rotations), new RoutingPolicy(owner, + ClusterSpec.Id.from("my-cluster2"), ZoneId.from("prod", "us-north-2"), - HostName.from("my-pretty-alias-2"), HostName.from("long-and-ugly-name-2"), Optional.empty(), rotations)); @@ -40,4 +43,30 @@ public class RoutingPolicySerializerTest { assertEquals(loadBalancers, serialized); } + @Test + public void test_legacy_serialization() { // TODO: Remove after 7.43 has been released + String json = "{\n" + + " \"routingPolicies\": [\n" + + " {\n" + + " \"alias\": \"my-pretty-alias\",\n" + + " \"zone\": \"prod.us-north-1\",\n" + + " \"canonicalName\": \"long-and-ugly-name\",\n" + + " \"dnsZone\": \"zone1\",\n" + + " \"rotations\": [\n" + + " \"r1\",\n" + + " \"r2\"\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + ApplicationId owner = ApplicationId.defaultId(); + Set<RoutingPolicy> expected = Set.of(new RoutingPolicy(owner, + ClusterSpec.Id.from("default"), + ZoneId.from("prod", "us-north-1"), + HostName.from("long-and-ugly-name"), + Optional.of("zone1"), + Set.of(RotationName.from("r1"), RotationName.from("r2")))); + assertEquals(expected, serializer.fromSlime(owner, SlimeUtils.jsonToSlime(json))); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index d9cdea2ea7b..c851cb18e8c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -4,7 +4,9 @@ package com.yahoo.vespa.hosted.controller.restapi; import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; @@ -15,23 +17,23 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; -import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; -import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; +import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; import java.io.File; import java.time.Duration; @@ -94,6 +96,28 @@ public class ContainerControllerTester { return application; } + public void deployCompletely(Application application, ApplicationPackage applicationPackage, long projectId, + boolean failStaging) { + jobCompletion(JobType.component).application(application) + .projectId(projectId) + .uploadArtifact(applicationPackage) + .submit(); + DeploymentSteps steps = controller().applications().deploymentTrigger().steps(applicationPackage.deploymentSpec()); + boolean succeeding = true; + for (var job : steps.jobs()) { + if (!succeeding) return; + var zone = job.zone(controller().system()); + deploy(application, applicationPackage, zone); + if (failStaging && zone.environment() == Environment.staging) { + succeeding = false; + } + if (zone.environment().isTest()) { + controller().applications().deactivate(application.id(), zone); + } + jobCompletion(job).application(application).success(succeeding).projectId(projectId).submit(); + } + } + /** Notify the controller about a job completing */ public BuildJob jobCompletion(JobType job) { return new BuildJob(this::notifyJobCompletion, artifactRepository()).type(job); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index 11ac250d4e0..ef86ffa125f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -6,11 +6,11 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Version; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.http.filter.FilterChainRepository; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index d3f0f423089..a7a28591d62 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.RotationName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -34,7 +35,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; @@ -1344,10 +1344,16 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void applicationWithPerClusterGlobalRotation() { + public void applicationWithRoutingPolicy() { Application app = controllerTester.createApplication(); - RoutingPolicy policy = new RoutingPolicy(app.id(), ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - ClusterSpec.Id.from("default"), controllerTester.controller().system(), + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .build(); + controllerTester.deployCompletely(app, applicationPackage, 1, false); + RoutingPolicy policy = new RoutingPolicy(app.id(), + ClusterSpec.Id.from("default"), + ZoneId.from(Environment.prod, RegionName.from("us-west-1")), HostName.from("lb-0-canonical-name"), Optional.of("dns-zone-1"), Set.of(RotationName.from("c0"))); tester.controller().curator().writeRoutingPolicies(app.id(), Set.of(policy)); @@ -1355,7 +1361,12 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET) .userIdentity(USER_ID), - new File("application-cluster-global-rotation.json")); + new File("application-with-routing-policy.json")); + + // GET deployment + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default", GET) + .userIdentity(USER_ID), + new File("deployment-with-routing-policy.json")); } private void notifyCompletion(DeploymentJobs.JobReport report, ContainerControllerTester tester) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json deleted file mode 100644 index baaf0cd038d..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "tenant": "tenant1", - "application": "application1", - "instance": "default", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", - "deployedInternally": false, - "deploymentJobs": [], - "changeBlockers": [], - "compileVersion": "(ignore)", - "globalRotations": [ - "https://c0.application1.tenant1.global.vespa.oath.cloud/" - ], - "instances": [], - "metrics": { - "queryServiceQuality": 0.0, - "writeServiceQuality": 0.0 - }, - "activity": {} -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-with-routing-policy.json new file mode 100644 index 00000000000..627afbf2674 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-with-routing-policy.json @@ -0,0 +1,203 @@ +{ + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + }, + "projectId": 1, + "deployedInternally": false, + "deploymentJobs": [ + { + "type": "component", + "success": true, + "lastCompleted": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Application commit", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Application commit", + "at": "(ignore)" + } + }, + { + "type": "system-test", + "success": true, + "lastTriggered": { + "id": -1, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing last changes outside prod", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing last changes outside prod", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing last changes outside prod", + "at": "(ignore)" + } + }, + { + "type": "staging-test", + "success": true, + "lastTriggered": { + "id": -1, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing deployment for production-us-west-1 (platform (ignore), application 1.0.42-commit1)", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing deployment for production-us-west-1 (platform (ignore), application 1.0.42-commit1)", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing deployment for production-us-west-1 (platform (ignore), application 1.0.42-commit1)", + "at": "(ignore)" + } + }, + { + "type": "production-us-west-1", + "success": true, + "lastTriggered": { + "id": -1, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "New change available", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "New change available", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "New change available", + "at": "(ignore)" + } + } + ], + "changeBlockers": [], + "compileVersion": "(ignore)", + "globalRotations": [ + "https://c0.application1.tenant1.global.vespa.oath.cloud/" + ], + "instances": [ + { + "environment": "prod", + "region": "us-west-1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default" + } + ], + "metrics": { + "queryServiceQuality": 0.0, + "writeServiceQuality": 0.0 + }, + "activity": {} +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json new file mode 100644 index 00000000000..519c9b1c842 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -0,0 +1,44 @@ +{ + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "environment": "prod", + "region": "us-west-1", + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://application1.tenant1.us-west-1.vespa.oath.cloud/" + } + ], + "serviceUrls": [ + "http://old-endpoint.vespa.yahooapis.com:4080", + "http://qrs-endpoint.vespa.yahooapis.com:4080", + "http://feeding-endpoint.vespa.yahooapis.com:4080", + "http://global-endpoint.vespa.yahooapis.com:4080", + "http://alias-endpoint.vespa.yahooapis.com:4080" + ], + "nodes": "http://localhost:8080/zone/v2/prod/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-west-1&application=tenant1.application1", + "version": "(ignore)", + "revision": "1.0.42-commit1", + "deployTimeEpochMs": "(ignore)", + "screwdriverId": "1", + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1", + "activity": {}, + "cost": { + "tco": 0, + "waste": 0, + "utilization": 0.0, + "cluster": {} + }, + "metrics": { + "queriesPerSecond": 0.0, + "writesPerSecond": 0.0, + "documentCount": 0.0, + "queryLatencyMillis": 0.0, + "writeLatencyMillis": 0.0 + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index af21260676c..6caac3bd532 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -4,6 +4,7 @@ "instance": "default", "environment": "prod", "region": "us-central-1", + "endpoints": [], "serviceUrls": [ "http://old-endpoint.vespa.yahooapis.com:4080", "http://qrs-endpoint.vespa.yahooapis.com:4080", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json index 54e94c4521e..67c71ee3880 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json @@ -4,6 +4,7 @@ "instance": "default", "environment": "dev", "region": "us-west-1", + "endpoints": [], "serviceUrls": [ "http://old-endpoint.vespa.yahooapis.com:4080", "http://qrs-endpoint.vespa.yahooapis.com:4080", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index cfefe629b9a..9b08fccf883 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -7,6 +7,7 @@ "instance": "default", "environment": "prod", "region": "us-central-1", + "endpoints": [], "serviceUrls": [ "http://old-endpoint.vespa.yahooapis.com:4080", "http://qrs-endpoint.vespa.yahooapis.com:4080", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 575427c9222..73977d7c2fa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -5,18 +5,15 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import org.junit.Before; import org.junit.Test; import java.io.File; @@ -31,13 +28,6 @@ public class DeploymentApiTest extends ControllerContainerTest { private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/"; - private ContainerControllerTester tester; - - @Before - public void before() { - tester = new ContainerControllerTester(container, responseFiles); - } - @Test public void testDeploymentApi() { ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); @@ -55,11 +45,11 @@ public class DeploymentApiTest extends ControllerContainerTest { "application2"); Application applicationWithoutDeployment = tester.createApplication("domain3", "tenant3", "application3"); - deployCompletely(failingApplication, applicationPackage, 1L, true); - deployCompletely(productionApplication, applicationPackage, 2L, true); + tester.deployCompletely(failingApplication, applicationPackage, 1L, false); + tester.deployCompletely(productionApplication, applicationPackage, 2L, false); // Deploy once so that job information is stored, then remove the deployment - deployCompletely(applicationWithoutDeployment, applicationPackage, 3L, true); + tester.deployCompletely(applicationWithoutDeployment, applicationPackage, 3L, false); tester.controller().applications().deactivate(applicationWithoutDeployment.id(), ZoneId.from("prod", "us-west-1")); // New version released @@ -70,8 +60,8 @@ public class DeploymentApiTest extends ControllerContainerTest { tester.upgrader().maintain(); tester.controller().applications().deploymentTrigger().triggerReadyJobs(); tester.controller().applications().deploymentTrigger().triggerReadyJobs(); - deployCompletely(failingApplication, applicationPackage, 1L, false); - deployCompletely(productionApplication, applicationPackage, 2L, true); + tester.deployCompletely(failingApplication, applicationPackage, 1L, true); + tester.deployCompletely(productionApplication, applicationPackage, 2L, false); tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()), tester.controller())); @@ -98,34 +88,4 @@ public class DeploymentApiTest extends ControllerContainerTest { return new VersionStatus(censored); } - private void deployCompletely(Application application, ApplicationPackage applicationPackage, long projectId, - boolean success) { - tester.jobCompletion(JobType.component) - .application(application) - .projectId(projectId) - .uploadArtifact(applicationPackage) - .submit(); - tester.deploy(application, applicationPackage, ZoneId.from(Environment.test, RegionName.from("us-east-1")) - ); - tester.jobCompletion(JobType.systemTest) - .application(application) - .projectId(projectId) - .submit(); - tester.deploy(application, applicationPackage, ZoneId.from(Environment.staging, RegionName.from("us-east-3")) - ); - tester.jobCompletion(JobType.stagingTest) - .application(application) - .projectId(projectId) - .success(success) - .submit(); - if (success) { - tester.deploy(application, applicationPackage, ZoneId.from(Environment.prod, - RegionName.from("us-west-1"))); - tester.jobCompletion(JobType.productionUsWest1) - .application(application) - .projectId(projectId) - .submit(); - } - } - } diff --git a/linguistics/src/main/java/com/yahoo/language/process/Transformer.java b/linguistics/src/main/java/com/yahoo/language/process/Transformer.java index 398ddc0262b..46f3c060d4e 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Transformer.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Transformer.java @@ -6,7 +6,7 @@ import com.yahoo.language.Language; /** * Interface for providers of text transformations such as accent removal. * - * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a> + * @author Mathias Mølster Lidal */ public interface Transformer { diff --git a/parent/pom.xml b/parent/pom.xml index d5e662ed7da..2c8e7264e23 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -122,6 +122,14 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> <version>3.1.1</version> + <dependencies> + <!-- TODO: Remove when upgrading to 3.1.2 --> + <dependency> + <groupId>org.apache.maven.shared</groupId> + <artifactId>maven-dependency-analyzer</artifactId> + <version>1.11.1</version> + </dependency> + </dependencies> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java index b4549fe495c..1fcd5d72a00 100644 --- a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java +++ b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java @@ -8,11 +8,14 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; /** * @author Simon Thoresen Hult @@ -20,7 +23,6 @@ import java.io.PrintStream; class FeederParams { enum DumpFormat {JSON, VESPA}; - private InputStream stdIn = System.in; private PrintStream stdErr = System.err; private PrintStream stdOut = System.out; private Route route = Route.parse("default"); @@ -30,14 +32,10 @@ class FeederParams { private boolean benchmarkMode = false; private int numDispatchThreads = 1; private int maxPending = 0; + private List<InputStream> inputStreams = new ArrayList<>(); - InputStream getStdIn() { - return stdIn; - } - - FeederParams setStdIn(InputStream stdIn) { - this.stdIn = stdIn; - return this; + FeederParams() { + inputStreams.add(System.in); } PrintStream getStdErr() { @@ -82,11 +80,6 @@ class FeederParams { return this; } - FeederParams setMaxPending(int maxPending) { - this.maxPending = maxPending; - return this; - } - boolean isSerialTransferEnabled() { return maxPending == 1; } @@ -96,6 +89,11 @@ class FeederParams { numDispatchThreads = 1; return this; } + List<InputStream> getInputStreams() { return inputStreams; } + FeederParams setInputStreams(List<InputStream> inputStreams) { + this.inputStreams = inputStreams; + return this; + } int getNumDispatchThreads() { return numDispatchThreads; } int getMaxPending() { return maxPending; } @@ -133,6 +131,13 @@ class FeederParams { setSerialTransfer(); } + if ( !cmd.getArgList().isEmpty()) { + inputStreams.clear(); + for (String fileName : cmd.getArgList()) { + inputStreams.add(new FileInputStream(new File(fileName))); + } + } + return this; } diff --git a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java index 36e5cc37ea5..32e883f171a 100644 --- a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java +++ b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java @@ -44,6 +44,8 @@ import java.io.OutputStream; import java.io.PrintStream; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -56,24 +58,59 @@ import java.util.concurrent.atomic.AtomicReference; */ public class SimpleFeeder implements ReplyHandler { - private final static long REPORT_INTERVAL = TimeUnit.SECONDS.toMillis(10); - private final static long HEADER_INTERVAL = REPORT_INTERVAL * 24; + private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); - private final InputStream in; + private final List<InputStream> inputStreams; private final PrintStream out; private final RPCMessageBus mbus; private final SourceSession session; + private final int numThreads; + private final Destination destination; + private final boolean benchmarkMode; + private final static long REPORT_INTERVAL = TimeUnit.SECONDS.toMillis(10); private final long startTime = System.currentTimeMillis(); private final AtomicReference<Throwable> failure = new AtomicReference<>(null); private final AtomicLong numReplies = new AtomicLong(0); private long maxLatency = Long.MIN_VALUE; private long minLatency = Long.MAX_VALUE; - private long nextHeader = startTime + HEADER_INTERVAL; private long nextReport = startTime + REPORT_INTERVAL; private long sumLatency = 0; - private final int numThreads; - private final Destination destination; - private final boolean benchmarkMode; + + static class Metrics { + + private final Destination destination; + private final FeedReader reader; + private final Executor executor; + AtomicReference<Throwable> failure; + + Metrics(Destination destination, FeedReader reader, Executor executor, AtomicReference<Throwable> failure) { + this.destination = destination; + this.reader = reader; + this.executor = executor; + this.failure = failure; + } + + long feed() throws Throwable { + long numMessages = 0; + while (failure.get() == null) { + FeedOperation op = reader.read(); + if (op.getType() == FeedOperation.Type.INVALID) { + break; + } + if (executor != null) { + executor.execute(() -> sendOperation(op)); + } else { + sendOperation(op); + } + ++numMessages; + } + return numMessages; + } + private void sendOperation(FeedOperation op) { + destination.send(op); + } + } + public static void main(String[] args) throws Throwable { new SimpleFeeder(new FeederParams().parseArgs(args)).run().close(); @@ -301,7 +338,7 @@ public class SimpleFeeder implements ReplyHandler { return new JsonDestination(params.getDumpStream(), failure, numReplies); } SimpleFeeder(FeederParams params) { - in = params.getStdIn(); + inputStreams = params.getInputStreams(); out = params.getStdOut(); numThreads = params.getNumDispatchThreads(); mbus = newMessageBus(docTypeMgr, params.getConfigId()); @@ -313,12 +350,8 @@ public class SimpleFeeder implements ReplyHandler { : new MbusDestination(session, params.getRoute(), failure, params.getStdErr()); } - private void sendOperation(FeedOperation op) { - destination.send(op); - } - SourceSession getSourceSession() { return session; } - private FeedReader createFeedReader() throws Exception { + private FeedReader createFeedReader(InputStream in) throws Exception { in.mark(8); byte [] b = new byte[2]; int numRead = readExact(in, b); @@ -335,6 +368,8 @@ public class SimpleFeeder implements ReplyHandler { } } + + SimpleFeeder run() throws Throwable { ExecutorService executor = (numThreads > 1) ? new ThreadPoolExecutor(numThreads, numThreads, 0L, TimeUnit.SECONDS, @@ -342,29 +377,19 @@ public class SimpleFeeder implements ReplyHandler { ThreadFactoryFactory.getDaemonThreadFactory("perf-feeder"), new ThreadPoolExecutor.CallerRunsPolicy()) : null; - FeedReader reader = createFeedReader(); - - printHeader(); - long numMessages = 0; - while (failure.get() == null) { - FeedOperation op = reader.read(); - if (op.getType() == FeedOperation.Type.INVALID) { - break; - } - if (executor != null) { - executor.execute(() -> sendOperation(op)); - } else { - sendOperation(op); - } - ++numMessages; + printHeader(out); + long numMessagesSent = 0; + for (InputStream in : inputStreams) { + Metrics m = new Metrics(destination, createFeedReader(in), executor, failure); + numMessagesSent += m.feed(); } - while (failure.get() == null && numReplies.get() < numMessages) { + while (failure.get() == null && numReplies.get() < numMessagesSent) { Thread.sleep(100); } if (failure.get() != null) { throw failure.get(); } - printReport(); + printReport(out); return this; } @@ -414,23 +439,18 @@ public class SimpleFeeder implements ReplyHandler { maxLatency = Math.max(maxLatency, latency); sumLatency += latency; if (benchmarkMode) { return; } - if (now > nextHeader) { - printHeader(); - nextHeader += HEADER_INTERVAL; - } if (now > nextReport) { - printReport(); + printReport(out); nextReport += REPORT_INTERVAL; } } - - private void printHeader() { + private static void printHeader(PrintStream out) { out.println("# Time used, num ok, num error, min latency, max latency, average latency"); } - private void printReport() { + private synchronized void printReport(PrintStream out) { out.format("%10d, %12d, %11d, %11d, %11d\n", System.currentTimeMillis() - startTime, - numReplies.get(), minLatency, maxLatency, sumLatency / numReplies.get()); + numReplies.get(), minLatency, maxLatency, sumLatency / Long.max(1, numReplies.get())); } private static String formatErrors(Reply reply) { diff --git a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java index d44cf41f9ab..5cc5d0bc018 100644 --- a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java +++ b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java @@ -5,12 +5,12 @@ import com.yahoo.messagebus.routing.Route; import org.apache.commons.cli.ParseException; import org.junit.Test; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.PrintStream; import static org.junit.Assert.assertEquals; @@ -24,18 +24,14 @@ import static org.junit.Assert.assertTrue; * @author Simon Thoresen Hult */ public class FeederParamsTest { - static final String TESTFILE_JSON = "test.json"; - static final String TESTFILE_VESPA = "test.vespa"; - static final String TESTFILE_UNKNOWN = "test.xyz"; + private static final String TESTFILE_JSON = "test.json"; + private static final String TESTFILE_VESPA = "test.vespa"; + private static final String TESTFILE_UNKNOWN = "test.xyz"; @Test public void requireThatAccessorsWork() { FeederParams params = new FeederParams(); - InputStream stdIn = new ByteArrayInputStream(new byte[1]); - params.setStdIn(stdIn); - assertSame(stdIn, params.getStdIn()); - PrintStream stdErr = new PrintStream(new ByteArrayOutputStream()); params.setStdErr(stdErr); assertSame(stdErr, params.getStdErr()); @@ -55,7 +51,7 @@ public class FeederParamsTest { @Test public void requireThatParamsHaveReasonableDefaults() { FeederParams params = new FeederParams(); - assertSame(System.in, params.getStdIn()); + assertSame(System.in, params.getInputStreams().get(0)); assertSame(System.err, params.getStdErr()); assertSame(System.out, params.getStdOut()); assertEquals(Route.parse("default"), params.getRoute()); @@ -66,16 +62,14 @@ public class FeederParamsTest { @Test public void requireThatSerialTransferOptionIsParsed() throws ParseException, FileNotFoundException { assertTrue(new FeederParams().parseArgs("-s").isSerialTransferEnabled()); - assertTrue(new FeederParams().parseArgs("foo", "-s").isSerialTransferEnabled()); - assertTrue(new FeederParams().parseArgs("-s", "foo").isSerialTransferEnabled()); assertTrue(new FeederParams().parseArgs("--serial").isSerialTransferEnabled()); - assertTrue(new FeederParams().parseArgs("foo", "--serial").isSerialTransferEnabled()); - assertTrue(new FeederParams().parseArgs("--serial", "foo").isSerialTransferEnabled()); + assertEquals(1, new FeederParams().parseArgs("-s").getMaxPending()); + assertEquals(1, new FeederParams().parseArgs("-s").getNumDispatchThreads()); } @Test public void requireThatArgumentsAreParsedAsRoute() throws ParseException, FileNotFoundException { - assertEquals(Route.parse("foo bar"), new FeederParams().parseArgs("-r foo bar").getRoute()); + assertEquals(Route.parse("foo bar"), new FeederParams().parseArgs("-r", "foo bar").getRoute()); assertEquals(Route.parse("foo bar"), new FeederParams().parseArgs("--route","foo bar").getRoute()); } @@ -115,4 +109,20 @@ public class FeederParamsTest { assertTrue(new File(TESTFILE_UNKNOWN).delete()); } + @Test + public void requireThatInputFilesAreAggregated() throws ParseException, IOException { + File json = new File(TESTFILE_JSON); + File vespa = new File(TESTFILE_VESPA); + new FileOutputStream(json).close(); + new FileOutputStream(vespa).close(); + FeederParams p = new FeederParams(); + p.parseArgs("-n", "3", TESTFILE_JSON, TESTFILE_VESPA); + assertEquals(3, p.getNumDispatchThreads()); + assertEquals(2, p.getInputStreams().size()); + assertTrue(p.getInputStreams().get(0) instanceof FileInputStream); + assertTrue(p.getInputStreams().get(1) instanceof FileInputStream); + json.delete(); + vespa.delete(); + } + } diff --git a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java index 2de7e831d04..8af4dd5dac9 100644 --- a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java +++ b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java @@ -22,6 +22,7 @@ import java.io.InputStream; import java.io.PrintStream; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.regex.Pattern; import static org.junit.Assert.assertEquals; @@ -313,7 +314,7 @@ public class SimpleFeederTest { server = new SimpleServer(CONFIG_DIR, validator); feeder = new SimpleFeeder(params.setConfigId("dir:" + CONFIG_DIR) .setStdErr(new PrintStream(err)) - .setStdIn(in) + .setInputStreams(Arrays.asList(in)) .setStdOut(new PrintStream(out))); } diff --git a/vespaclient-core/src/main/resources/configdefinitions/feeder.def b/vespaclient-core/src/main/resources/configdefinitions/feeder.def index e138be715d5..9462943ce03 100644 --- a/vespaclient-core/src/main/resources/configdefinitions/feeder.def +++ b/vespaclient-core/src/main/resources/configdefinitions/feeder.def @@ -8,6 +8,9 @@ abortondocumenterror bool default=true ## Whether or not to abort if there are errors sending messages to Vespa abortonsenderror bool default=true +## Prefix each document id with this string. +idprefix string default="" + ## Max number of pending operations. maxpendingdocs int default=0 @@ -17,6 +20,9 @@ maxfeedrate double default=0.0 ## Whether or not retrying is enabled. retryenabled bool default=true +## Delay between retries. +retrydelay double default=1 + ## Timeout for messagebus operations. timeout double default=180 diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java index a231582fb5d..d659c5b578f 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java @@ -124,10 +124,14 @@ public class Arguments { files.add(getParam(args, arg)); } else if ("--maxpending".equals(arg)) { feederConfigBuilder.maxpendingdocs(Integer.parseInt(getParam(args, arg))); + } else if ("--maxpendingsize".equals(arg)) { + feederConfigBuilder.maxpendingbytes(Integer.parseInt(getParam(args, arg))); } else if ("--mode".equals(arg)) { mode = getParam(args, arg); } else if ("--noretry".equals(arg)) { feederConfigBuilder.retryenabled(false); + } else if ("--retrydelay".equals(arg)) { + feederConfigBuilder.retrydelay(Integer.parseInt(getParam(args, arg))); } else if ("--route".equals(arg)) { feederConfigBuilder.route(getParam(args, arg)); } else if ("--timeout".equals(arg)) { diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 43388e4e18d..4f81f3baea8 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -706,27 +706,77 @@ ], "fields": [] }, - "com.yahoo.tensor.IndexedTensor$BoundBuilder": { - "superClass": "com.yahoo.tensor.IndexedTensor$Builder", + "com.yahoo.tensor.IndexedDoubleTensor$BoundDoubleBuilder": { + "superClass": "com.yahoo.tensor.IndexedTensor$BoundBuilder", "interfaces": [], "attributes": [ "public" ], "methods": [ + "public varargs com.yahoo.tensor.IndexedTensor$BoundBuilder cell(float, long[])", "public varargs com.yahoo.tensor.IndexedTensor$BoundBuilder cell(double, long[])", "public com.yahoo.tensor.Tensor$Builder$CellBuilder cell()", + "public com.yahoo.tensor.IndexedTensor$Builder cell(com.yahoo.tensor.TensorAddress, float)", "public com.yahoo.tensor.IndexedTensor$Builder cell(com.yahoo.tensor.TensorAddress, double)", "public com.yahoo.tensor.IndexedTensor build()", + "public com.yahoo.tensor.IndexedTensor$Builder cell(com.yahoo.tensor.Tensor$Cell, float)", "public com.yahoo.tensor.IndexedTensor$Builder cell(com.yahoo.tensor.Tensor$Cell, double)", + "public void cellByDirectIndex(long, float)", "public void cellByDirectIndex(long, double)", + "public bridge synthetic com.yahoo.tensor.IndexedTensor$Builder cell(float, long[])", "public bridge synthetic com.yahoo.tensor.IndexedTensor$Builder cell(double, long[])", "public bridge synthetic com.yahoo.tensor.Tensor build()", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.Tensor$Cell, float)", "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.Tensor$Cell, double)", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(float, long[])", "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(double, long[])", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, float)", "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, double)" ], "fields": [] }, + "com.yahoo.tensor.IndexedFloatTensor$BoundFloatBuilder": { + "superClass": "com.yahoo.tensor.IndexedTensor$BoundBuilder", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public varargs com.yahoo.tensor.IndexedTensor$BoundBuilder cell(double, long[])", + "public varargs com.yahoo.tensor.IndexedTensor$BoundBuilder cell(float, long[])", + "public com.yahoo.tensor.Tensor$Builder$CellBuilder cell()", + "public com.yahoo.tensor.IndexedTensor$Builder cell(com.yahoo.tensor.TensorAddress, double)", + "public com.yahoo.tensor.IndexedTensor$Builder cell(com.yahoo.tensor.TensorAddress, float)", + "public com.yahoo.tensor.IndexedTensor build()", + "public com.yahoo.tensor.IndexedTensor$Builder cell(com.yahoo.tensor.Tensor$Cell, double)", + "public com.yahoo.tensor.IndexedTensor$Builder cell(com.yahoo.tensor.Tensor$Cell, float)", + "public void cellByDirectIndex(long, double)", + "public void cellByDirectIndex(long, float)", + "public bridge synthetic com.yahoo.tensor.IndexedTensor$Builder cell(float, long[])", + "public bridge synthetic com.yahoo.tensor.IndexedTensor$Builder cell(double, long[])", + "public bridge synthetic com.yahoo.tensor.Tensor build()", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.Tensor$Cell, float)", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.Tensor$Cell, double)", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(float, long[])", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(double, long[])", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, float)", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, double)" + ], + "fields": [] + }, + "com.yahoo.tensor.IndexedTensor$BoundBuilder": { + "superClass": "com.yahoo.tensor.IndexedTensor$Builder", + "interfaces": [], + "attributes": [ + "public", + "abstract" + ], + "methods": [ + "public abstract void cellByDirectIndex(long, double)", + "public abstract void cellByDirectIndex(long, float)" + ], + "fields": [] + }, "com.yahoo.tensor.IndexedTensor$Builder": { "superClass": "java.lang.Object", "interfaces": [ @@ -740,9 +790,11 @@ "public static com.yahoo.tensor.IndexedTensor$Builder of(com.yahoo.tensor.TensorType)", "public static com.yahoo.tensor.IndexedTensor$Builder of(com.yahoo.tensor.TensorType, com.yahoo.tensor.DimensionSizes)", "public varargs abstract com.yahoo.tensor.IndexedTensor$Builder cell(double, long[])", + "public varargs abstract com.yahoo.tensor.IndexedTensor$Builder cell(float, long[])", "public com.yahoo.tensor.TensorType type()", "public abstract com.yahoo.tensor.IndexedTensor build()", "public bridge synthetic com.yahoo.tensor.Tensor build()", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(float, long[])", "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(double, long[])" ], "fields": [] @@ -792,25 +844,26 @@ "com.yahoo.tensor.Tensor" ], "attributes": [ - "public" + "public", + "abstract" ], "methods": [ - "public long size()", "public java.util.Iterator cellIterator()", "public com.yahoo.tensor.IndexedTensor$SubspaceIterator cellIterator(com.yahoo.tensor.PartialAddress, com.yahoo.tensor.DimensionSizes)", "public java.util.Iterator valueIterator()", "public java.util.Iterator subspaceIterator(java.util.Set, com.yahoo.tensor.DimensionSizes)", "public java.util.Iterator subspaceIterator(java.util.Set)", "public varargs double get(long[])", + "public varargs float getFloat(long[])", "public double get(com.yahoo.tensor.TensorAddress)", - "public double get(long)", + "public abstract double get(long)", + "public abstract float getFloat(long)", "public com.yahoo.tensor.TensorType type()", - "public com.yahoo.tensor.IndexedTensor withType(com.yahoo.tensor.TensorType)", + "public abstract com.yahoo.tensor.IndexedTensor withType(com.yahoo.tensor.TensorType)", "public com.yahoo.tensor.DimensionSizes dimensionSizes()", "public java.util.Map cells()", "public com.yahoo.tensor.Tensor merge(java.util.function.DoubleBinaryOperator, java.util.Map)", "public com.yahoo.tensor.Tensor remove(java.util.Set)", - "public int hashCode()", "public java.lang.String toString()", "public boolean equals(java.lang.Object)", "public bridge synthetic com.yahoo.tensor.Tensor withType(com.yahoo.tensor.TensorType)" @@ -829,11 +882,15 @@ "public static com.yahoo.tensor.MappedTensor$Builder of(com.yahoo.tensor.TensorType)", "public com.yahoo.tensor.Tensor$Builder$CellBuilder cell()", "public com.yahoo.tensor.TensorType type()", + "public com.yahoo.tensor.MappedTensor$Builder cell(com.yahoo.tensor.TensorAddress, float)", "public com.yahoo.tensor.MappedTensor$Builder cell(com.yahoo.tensor.TensorAddress, double)", + "public varargs com.yahoo.tensor.MappedTensor$Builder cell(float, long[])", "public varargs com.yahoo.tensor.MappedTensor$Builder cell(double, long[])", "public com.yahoo.tensor.MappedTensor build()", "public bridge synthetic com.yahoo.tensor.Tensor build()", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(float, long[])", "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(double, long[])", + "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, float)", "public bridge synthetic com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, double)" ], "fields": [] @@ -870,6 +927,7 @@ ], "methods": [ "public long denseSubspaceSize()", + "public com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, float)", "public com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, double)", "public com.yahoo.tensor.Tensor$Builder block(com.yahoo.tensor.TensorAddress, double[])", "public com.yahoo.tensor.MixedTensor build()", @@ -889,6 +947,7 @@ "methods": [ "public static com.yahoo.tensor.MixedTensor$Builder of(com.yahoo.tensor.TensorType)", "public com.yahoo.tensor.TensorType type()", + "public varargs com.yahoo.tensor.Tensor$Builder cell(float, long[])", "public varargs com.yahoo.tensor.Tensor$Builder cell(double, long[])", "public com.yahoo.tensor.Tensor$Builder$CellBuilder cell()", "public abstract com.yahoo.tensor.MixedTensor build()", @@ -917,6 +976,7 @@ "public" ], "methods": [ + "public com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, float)", "public com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, double)", "public com.yahoo.tensor.MixedTensor build()", "public void trackBounds(com.yahoo.tensor.TensorAddress)", @@ -982,7 +1042,8 @@ "methods": [ "public com.yahoo.tensor.Tensor$Builder$CellBuilder label(java.lang.String, java.lang.String)", "public com.yahoo.tensor.Tensor$Builder$CellBuilder label(java.lang.String, long)", - "public com.yahoo.tensor.Tensor$Builder value(double)" + "public com.yahoo.tensor.Tensor$Builder value(double)", + "public com.yahoo.tensor.Tensor$Builder value(float)" ], "fields": [] }, @@ -1000,8 +1061,11 @@ "public abstract com.yahoo.tensor.TensorType type()", "public abstract com.yahoo.tensor.Tensor$Builder$CellBuilder cell()", "public abstract com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, double)", + "public abstract com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, float)", "public varargs abstract com.yahoo.tensor.Tensor$Builder cell(double, long[])", + "public varargs abstract com.yahoo.tensor.Tensor$Builder cell(float, long[])", "public com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.Tensor$Cell, double)", + "public com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.Tensor$Cell, float)", "public abstract com.yahoo.tensor.Tensor build()" ], "fields": [] @@ -1017,6 +1081,8 @@ "methods": [ "public com.yahoo.tensor.TensorAddress getKey()", "public java.lang.Double getValue()", + "public float getFloatValue()", + "public double getDoubleValue()", "public java.lang.Double setValue(java.lang.Double)", "public boolean equals(java.lang.Object)", "public int hashCode()", diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedDoubleTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedDoubleTensor.java new file mode 100644 index 00000000000..285837a1bc6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedDoubleTensor.java @@ -0,0 +1,112 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.Arrays; + +/** + * An indexed tensor implementation holding values as doubles + * + * @author bratseth + */ +class IndexedDoubleTensor extends IndexedTensor { + + private final double[] values; + + IndexedDoubleTensor(TensorType type, DimensionSizes dimensionSizes, double[] values) { + super(type, dimensionSizes); + this.values = values; + } + + @Override + public long size() { + return values.length; + } + + @Override + public double get(long valueIndex) { return values[(int)valueIndex]; } + + @Override + public float getFloat(long valueIndex) { return (float)get(valueIndex); } + + @Override + public IndexedTensor withType(TensorType type) { + throwOnIncompatibleType(type); + return new IndexedDoubleTensor(type, dimensionSizes(), values); + } + + @Override + public int hashCode() { return Arrays.hashCode(values); } + + /** A bound builder can create the double array directly */ + public static class BoundDoubleBuilder extends BoundBuilder { + + private double[] values; + + BoundDoubleBuilder(TensorType type, DimensionSizes sizes) { + super(type, sizes); + values = new double[(int)sizes.totalSize()]; + } + + @Override + public IndexedTensor.BoundBuilder cell(float value, long ... indexes) { + return cell((double)value, indexes); + } + + @Override + public IndexedTensor.BoundBuilder cell(double value, long ... indexes) { + values[(int)toValueIndex(indexes, sizes())] = value; + return this; + } + + @Override + public CellBuilder cell() { + return new CellBuilder(type, this); + } + + @Override + public Builder cell(TensorAddress address, float value) { + return cell(address, (double)value); + } + + @Override + public Builder cell(TensorAddress address, double value) { + values[(int)toValueIndex(address, sizes())] = value; + return this; + } + + @Override + public IndexedTensor build() { + IndexedTensor tensor = new IndexedDoubleTensor(type, sizes(), values); + // prevent further modification + values = null; + return tensor; + } + + @Override + public Builder cell(Cell cell, float value) { + return cell(cell, (double)value); + } + + @Override + public Builder cell(Cell cell, double value) { + long directIndex = cell.getDirectIndex(); + if (directIndex >= 0) // optimization + values[(int)directIndex] = value; + else + super.cell(cell, value); + return this; + } + + @Override + public void cellByDirectIndex(long index, float value) { + cellByDirectIndex(index, (double)value); + } + + @Override + public void cellByDirectIndex(long index, double value) { + values[(int)index] = value; + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedFloatTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedFloatTensor.java new file mode 100644 index 00000000000..8f8c24c8421 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedFloatTensor.java @@ -0,0 +1,112 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.Arrays; + +/** + * An indexed tensor implementation holding values as floats + * + * @author bratseth + */ +class IndexedFloatTensor extends IndexedTensor { + + private final float[] values; + + IndexedFloatTensor(TensorType type, DimensionSizes dimensionSizes, float[] values) { + super(type, dimensionSizes); + this.values = values; + } + + @Override + public long size() { + return values.length; + } + + @Override + public double get(long valueIndex) { return getFloat(valueIndex); } + + @Override + public float getFloat(long valueIndex) { return values[(int)valueIndex]; } + + @Override + public IndexedTensor withType(TensorType type) { + throwOnIncompatibleType(type); + return new IndexedFloatTensor(type, dimensionSizes(), values); + } + + @Override + public int hashCode() { return Arrays.hashCode(values); } + + /** A bound builder can create the float array directly */ + public static class BoundFloatBuilder extends BoundBuilder { + + private float[] values; + + BoundFloatBuilder(TensorType type, DimensionSizes sizes) { + super(type, sizes); + values = new float[(int)sizes.totalSize()]; + } + + @Override + public IndexedTensor.BoundBuilder cell(double value, long ... indexes) { + return cell((float)value, indexes); + } + + @Override + public IndexedTensor.BoundBuilder cell(float value, long ... indexes) { + values[(int)toValueIndex(indexes, sizes())] = value; + return this; + } + + @Override + public CellBuilder cell() { + return new CellBuilder(type, this); + } + + @Override + public Builder cell(TensorAddress address, double value) { + return cell(address, (float)value); + } + + @Override + public Builder cell(TensorAddress address, float value) { + values[(int)toValueIndex(address, sizes())] = value; + return this; + } + + @Override + public IndexedTensor build() { + IndexedTensor tensor = new IndexedFloatTensor(type, sizes(), values); + // prevent further modification + values = null; + return tensor; + } + + @Override + public Builder cell(Cell cell, double value) { + return cell(cell, (float)value); + } + + @Override + public Builder cell(Cell cell, float value) { + long directIndex = cell.getDirectIndex(); + if (directIndex >= 0) // optimization + values[(int)directIndex] = value; + else + super.cell(cell, value); + return this; + } + + @Override + public void cellByDirectIndex(long index, double value) { + cellByDirectIndex(index, (float)value); + } + + @Override + public void cellByDirectIndex(long index, float value) { + values[(int)index] = value; + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index 38d832d01c2..19edfc0269e 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -20,7 +20,7 @@ import java.util.function.DoubleBinaryOperator; * * @author bratseth */ -public class IndexedTensor implements Tensor { +public abstract class IndexedTensor implements Tensor { /** The prescribed and possibly abstract type this is an instance of */ private final TensorType type; @@ -28,17 +28,9 @@ public class IndexedTensor implements Tensor { /** The sizes of the dimensions of this in the order of the dimensions of the type */ private final DimensionSizes dimensionSizes; - private final double[] values; - - private IndexedTensor(TensorType type, DimensionSizes dimensionSizes, double[] values) { + IndexedTensor(TensorType type, DimensionSizes dimensionSizes) { this.type = type; this.dimensionSizes = dimensionSizes; - this.values = values; - } - - @Override - public long size() { - return values.length; } /** @@ -96,13 +88,23 @@ public class IndexedTensor implements Tensor { } /** - * Returns the value at the given indexes + * Returns the value at the given indexes as a double * * @param indexes the indexes into the dimensions of this. Must be one number per dimension of this * @throws IndexOutOfBoundsException if any of the indexes are out of bound or a wrong number of indexes are given */ public double get(long ... indexes) { - return values[(int)toValueIndex(indexes, dimensionSizes)]; + return get((int)toValueIndex(indexes, dimensionSizes)); + } + + /** + * Returns the value at the given indexes as a float + * + * @param indexes the indexes into the dimensions of this. Must be one number per dimension of this + * @throws IndexOutOfBoundsException if any of the indexes are out of bound or a wrong number of indexes are given + */ + public float getFloat(long ... indexes) { + return getFloat((int)toValueIndex(indexes, dimensionSizes)); } /** Returns the value at this address, or NaN if there is no value at this address */ @@ -110,7 +112,7 @@ public class IndexedTensor implements Tensor { public double get(TensorAddress address) { // optimize for fast lookup within bounds: try { - return values[(int)toValueIndex(address, dimensionSizes)]; + return get((int)toValueIndex(address, dimensionSizes)); } catch (IndexOutOfBoundsException e) { return Double.NaN; @@ -118,15 +120,24 @@ public class IndexedTensor implements Tensor { } /** - * Returns the value at the given index by direct lookup. Only use + * Returns the value at the given index as a double by direct lookup. Only use * if you know the underlying data layout. * * @param valueIndex the direct index into the underlying data. * @throws IndexOutOfBoundsException if index is out of bounds */ - public double get(long valueIndex) { return values[(int)valueIndex]; } + public abstract double get(long valueIndex); - private static long toValueIndex(long[] indexes, DimensionSizes sizes) { + /** + * Returns the value at the given index as a float by direct lookup. Only use + * if you know the underlying data layout. + * + * @param valueIndex the direct index into the underlying data. + * @throws IndexOutOfBoundsException if index is out of bounds + */ + public abstract float getFloat(long valueIndex); + + static long toValueIndex(long[] indexes, DimensionSizes sizes) { if (indexes.length == 1) return indexes[0]; // for speed if (indexes.length == 0) return 0; // for speed @@ -140,7 +151,7 @@ public class IndexedTensor implements Tensor { return valueIndex; } - private static long toValueIndex(TensorAddress address, DimensionSizes sizes) { + static long toValueIndex(TensorAddress address, DimensionSizes sizes) { if (address.isEmpty()) return 0; long valueIndex = 0; @@ -160,17 +171,17 @@ public class IndexedTensor implements Tensor { return product; } + void throwOnIncompatibleType(TensorType type) { + if ( ! this.type().isRenamableTo(type)) + throw new IllegalArgumentException("Can not change type from " + this.type() + " to " + type + + ": Types are not compatible"); + } + @Override public TensorType type() { return type; } @Override - public IndexedTensor withType(TensorType type) { - if (!this.type.isRenamableTo(type)) { - throw new IllegalArgumentException("IndexedTensor.withType: types are not compatible. Current type: '" + - this.type.toString() + "', requested type: '" + type.toString() + "'"); - } - return new IndexedTensor(type, dimensionSizes, values); - } + public abstract IndexedTensor withType(TensorType type); public DimensionSizes dimensionSizes() { return dimensionSizes; @@ -179,13 +190,13 @@ public class IndexedTensor implements Tensor { @Override public Map<TensorAddress, Double> cells() { if (dimensionSizes.dimensions() == 0) - return Collections.singletonMap(TensorAddress.of(), values[0]); + return Collections.singletonMap(TensorAddress.of(), get(0)); ImmutableMap.Builder<TensorAddress, Double> builder = new ImmutableMap.Builder<>(); - Indexes indexes = Indexes.of(dimensionSizes, dimensionSizes, values.length); - for (long i = 0; i < values.length; i++) { + Indexes indexes = Indexes.of(dimensionSizes, dimensionSizes, size()); + for (long i = 0; i < size(); i++) { indexes.next(); - builder.put(indexes.toAddress(), values[(int)i]); + builder.put(indexes.toAddress(), get(i)); } return builder.build(); } @@ -201,9 +212,6 @@ public class IndexedTensor implements Tensor { } @Override - public int hashCode() { return Arrays.hashCode(values); } - - @Override public String toString() { return Tensor.toStandardString(this); } @Override @@ -222,7 +230,7 @@ public class IndexedTensor implements Tensor { public static Builder of(TensorType type) { if (type.dimensions().stream().allMatch(d -> d instanceof TensorType.IndexedBoundDimension)) - return new BoundBuilder(type); + return of(type, BoundBuilder.dimensionSizesOf(type)); else return new UnboundBuilder(type); } @@ -235,8 +243,8 @@ public class IndexedTensor implements Tensor { public static Builder of(TensorType type, DimensionSizes sizes) { // validate if (sizes.dimensions() != type.dimensions().size()) - throw new IllegalArgumentException(sizes.dimensions() + " is the wrong number of dimensions " + - "for " + type); + throw new IllegalArgumentException(sizes.dimensions() + + " is the wrong number of dimensions for " + type); for (int i = 0; i < sizes.dimensions(); i++ ) { Optional<Long> size = type.dimensions().get(i).size(); if (size.isPresent() && size.get() < sizes.size(i)) @@ -245,10 +253,16 @@ public class IndexedTensor implements Tensor { " but cannot be larger than " + size.get() + " in " + type); } - return new BoundBuilder(type, sizes); + if (type.valueType() == TensorType.Value.FLOAT) + return new IndexedFloatTensor.BoundFloatBuilder(type, sizes); + else if (type.valueType() == TensorType.Value.DOUBLE) + return new IndexedDoubleTensor.BoundDoubleBuilder(type, sizes); + else + return new IndexedDoubleTensor.BoundDoubleBuilder(type, sizes); // Default } public abstract Builder cell(double value, long ... indexes); + public abstract Builder cell(float value, long ... indexes); @Override public TensorType type() { return type; } @@ -259,74 +273,29 @@ public class IndexedTensor implements Tensor { } /** A bound builder can create the double array directly */ - public static class BoundBuilder extends Builder { + public static abstract class BoundBuilder extends Builder { private DimensionSizes sizes; - private double[] values; - private BoundBuilder(TensorType type) { - this(type, dimensionSizesOf(type)); - } - - static DimensionSizes dimensionSizesOf(TensorType type) { + private static DimensionSizes dimensionSizesOf(TensorType type) { DimensionSizes.Builder b = new DimensionSizes.Builder(type.dimensions().size()); for (int i = 0; i < type.dimensions().size(); i++) b.set(i, type.dimensions().get(i).size().get()); return b.build(); } - private BoundBuilder(TensorType type, DimensionSizes sizes) { + BoundBuilder(TensorType type, DimensionSizes sizes) { super(type); if ( sizes.dimensions() != type.dimensions().size()) throw new IllegalArgumentException("Must have a dimension size entry for each dimension in " + type); this.sizes = sizes; - values = new double[(int)sizes.totalSize()]; } - @Override - public BoundBuilder cell(double value, long ... indexes) { - values[(int)toValueIndex(indexes, sizes)] = value; - return this; - } - - @Override - public CellBuilder cell() { - return new CellBuilder(type, this); - } - - @Override - public Builder cell(TensorAddress address, double value) { - values[(int)toValueIndex(address, sizes)] = value; - return this; - } - - @Override - public IndexedTensor build() { - IndexedTensor tensor = new IndexedTensor(type, sizes, values); - // prevent further modification - sizes = null; - values = null; - return tensor; - } + DimensionSizes sizes() { return sizes; } - @Override - public Builder cell(Cell cell, double value) { - long directIndex = cell.getDirectIndex(); - if (directIndex >= 0) // optimization - values[(int)directIndex] = value; - else - super.cell(cell, value); - return this; - } + public abstract void cellByDirectIndex(long index, double value); - /** - * Set a cell value by the index in the internal layout of this cell. - * This requires knowledge of the internal layout of cells in this implementation, and should therefore - * probably not be used (but when it can be used it is fast). - */ - public void cellByDirectIndex(long index, double value) { - values[(int)index] = value; - } + public abstract void cellByDirectIndex(long index, float value); } @@ -348,12 +317,12 @@ public class IndexedTensor implements Tensor { if (firstDimension == null) throw new IllegalArgumentException("Tensor of type " + type() + " has no values"); if (type.dimensions().isEmpty()) // single number - return new IndexedTensor(type, new DimensionSizes.Builder(type.dimensions().size()).build(), new double[] {(Double) firstDimension.get(0) }); + return new IndexedDoubleTensor(type, new DimensionSizes.Builder(type.dimensions().size()).build(), new double[] {(Double) firstDimension.get(0) }); DimensionSizes dimensionSizes = findDimensionSizes(firstDimension); double[] values = new double[(int)dimensionSizes.totalSize()]; fillValues(0, 0, firstDimension, dimensionSizes, values); - return new IndexedTensor(type, dimensionSizes, values); + return new IndexedDoubleTensor(type, dimensionSizes, values); } private DimensionSizes findDimensionSizes(List<Object> firstDimension) { @@ -406,6 +375,11 @@ public class IndexedTensor implements Tensor { } @Override + public Builder cell(TensorAddress address, float value) { + return cell(address, (double)value); + } + + @Override public Builder cell(TensorAddress address, double value) { long[] indexes = new long[address.size()]; for (int i = 0; i < address.size(); i++) { @@ -415,6 +389,11 @@ public class IndexedTensor implements Tensor { return this; } + @Override + public Builder cell(float value, long... indexes) { + return cell((double)value, indexes); + } + /** * Set a value using an index API. The number of indexes must be the same as the dimensions in the type of this. * Values can be written in any order but all values needed to make this dense must be provided @@ -460,7 +439,7 @@ public class IndexedTensor implements Tensor { private final class CellIterator implements Iterator<Cell> { private long count = 0; - private final Indexes indexes = Indexes.of(dimensionSizes, dimensionSizes, values.length); + private final Indexes indexes = Indexes.of(dimensionSizes, dimensionSizes, size()); private final LazyCell reusedCell = new LazyCell(indexes, Double.NaN); @Override @@ -485,13 +464,13 @@ public class IndexedTensor implements Tensor { @Override public boolean hasNext() { - return count < values.length; + return count < size(); } @Override public Double next() { try { - return values[(int)count++]; + return get(count++); } catch (IndexOutOfBoundsException e) { throw new NoSuchElementException("No element at position " + count); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java index 22ceed22d3e..693c4b5f2b0 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java @@ -115,12 +115,22 @@ public class MappedTensor implements Tensor { public TensorType type() { return type; } @Override + public Builder cell(TensorAddress address, float value) { + return cell(address, (double)value); + } + + @Override public Builder cell(TensorAddress address, double value) { cells.put(address, value); return this; } @Override + public Builder cell(float value, long... labels) { + return cell((double)value, labels); + } + + @Override public Builder cell(double value, long... labels) { cells.put(TensorAddress.of(labels), value); return this; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java index c06cb2a0986..95f64cec0c1 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java @@ -193,6 +193,11 @@ public class MixedTensor implements Tensor { } @Override + public Tensor.Builder cell(float value, long... labels) { + return cell((double)value, labels); + } + + @Override public Tensor.Builder cell(double value, long... labels) { throw new UnsupportedOperationException("Not implemented."); } @@ -236,6 +241,11 @@ public class MixedTensor implements Tensor { } @Override + public Tensor.Builder cell(TensorAddress address, float value) { + return cell(address, (double)value); + } + + @Override public Tensor.Builder cell(TensorAddress address, double value) { TensorAddress sparsePart = index.sparsePartialAddress(address); long denseOffset = index.denseOffset(address); @@ -293,6 +303,11 @@ public class MixedTensor implements Tensor { } @Override + public Tensor.Builder cell(TensorAddress address, float value) { + return cell(address, (double)value); + } + + @Override public Tensor.Builder cell(TensorAddress address, double value) { cells.put(address, value); trackBounds(address); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index eb16801c306..ebb341147cf 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -370,9 +370,9 @@ public interface Tensor { class Cell implements Map.Entry<TensorAddress, Double> { private final TensorAddress address; - private final Double value; + private final Number value; - Cell(TensorAddress address, Double value) { + Cell(TensorAddress address, Number value) { this.address = address; this.value = value; } @@ -387,8 +387,15 @@ public interface Tensor { */ long getDirectIndex() { return -1; } + /** Returns the value as a double */ @Override - public Double getValue() { return value; } + public Double getValue() { return value.doubleValue(); } + + /** Returns the value as a float */ + public float getFloatValue() { return value.floatValue(); } + + /** Returns the value as a double */ + public double getDoubleValue() { return value.doubleValue(); } @Override public Double setValue(Double value) { @@ -446,9 +453,11 @@ public interface Tensor { /** Add a cell */ Builder cell(TensorAddress address, double value); + Builder cell(TensorAddress address, float value); /** Add a cell */ Builder cell(double value, long ... labels); + Builder cell(float value, long ... labels); /** * Add a cell @@ -459,6 +468,9 @@ public interface Tensor { default Builder cell(Cell cell, double value) { return cell(cell.getKey(), value); } + default Builder cell(Cell cell, float value) { + return cell(cell.getKey(), value); + } Tensor build(); @@ -484,6 +496,9 @@ public interface Tensor { public Builder value(double cellValue) { return tensorBuilder.cell(addressBuilder.build(), cellValue); } + public Builder value(float cellValue) { + return tensorBuilder.cell(addressBuilder.build(), cellValue); + } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index df78f3dfc3a..b1c7a2341c0 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -143,6 +143,7 @@ public class TensorType { } private boolean isConvertibleOrAssignableTo(TensorType generalization, boolean convertible, boolean considerName) { + if ( this.valueType() != generalization.valueType()) return false; // TODO: This can be relaxed if (generalization.dimensions().size() != this.dimensions().size()) return false; for (int i = 0; i < generalization.dimensions().size(); i++) { Dimension thisDimension = this.dimensions().get(i); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java index 5072484567d..0cec09157fb 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java @@ -38,7 +38,7 @@ public class DenseBinaryFormat implements BinaryFormat { if ( ! ( tensor instanceof IndexedTensor)) throw new RuntimeException("The dense format is only supported for indexed tensors"); encodeDimensions(buffer, (IndexedTensor)tensor); - encodeCells(buffer, tensor); + encodeCells(buffer, (IndexedTensor)tensor); } private void encodeDimensions(GrowableByteBuffer buffer, IndexedTensor tensor) { @@ -49,18 +49,21 @@ public class DenseBinaryFormat implements BinaryFormat { } } - private void encodeCells(GrowableByteBuffer buffer, Tensor tensor) { + private void encodeCells(GrowableByteBuffer buffer, IndexedTensor tensor) { switch (serializationValueType) { - case DOUBLE: encodeCells(tensor, buffer::putDouble); break; - case FLOAT: encodeCells(tensor, (i) -> buffer.putFloat(i.floatValue())); break; + case DOUBLE: encodeDoubleCells(tensor, buffer); break; + case FLOAT: encodeFloatCells(tensor, buffer); break; } } - private void encodeCells(Tensor tensor, Consumer<Double> consumer) { - Iterator<Double> i = tensor.valueIterator(); - while (i.hasNext()) { - consumer.accept(i.next()); - } + private void encodeDoubleCells(IndexedTensor tensor, GrowableByteBuffer buffer) { + for (int i = 0; i < tensor.size(); i++) + buffer.putDouble(tensor.get(i)); + } + + private void encodeFloatCells(IndexedTensor tensor, GrowableByteBuffer buffer) { + for (int i = 0; i < tensor.size(); i++) + buffer.putFloat(tensor.getFloat(i)); } @Override @@ -106,14 +109,19 @@ public class DenseBinaryFormat implements BinaryFormat { private void decodeCells(DimensionSizes sizes, GrowableByteBuffer buffer, IndexedTensor.BoundBuilder builder) { switch (serializationValueType) { - case DOUBLE: decodeCells(sizes, builder, buffer::getDouble); break; - case FLOAT: decodeCells(sizes, builder, () -> (double)buffer.getFloat()); break; + case DOUBLE: decodeDoubleCells(sizes, builder, buffer); break; + case FLOAT: decodeFloatCells(sizes, builder, buffer); break; } } - private void decodeCells(DimensionSizes sizes, IndexedTensor.BoundBuilder builder, Supplier<Double> supplier) { + private void decodeDoubleCells(DimensionSizes sizes, IndexedTensor.BoundBuilder builder, GrowableByteBuffer buffer) { + for (long i = 0; i < sizes.totalSize(); i++) + builder.cellByDirectIndex(i, buffer.getDouble()); + } + + private void decodeFloatCells(DimensionSizes sizes, IndexedTensor.BoundBuilder builder, GrowableByteBuffer buffer) { for (long i = 0; i < sizes.totalSize(); i++) - builder.cellByDirectIndex(i, supplier.get()); + builder.cellByDirectIndex(i, buffer.getFloat()); } } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index 02d16e6f3e4..b01d171792c 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -37,6 +37,17 @@ public class TensorTestCase { } @Test + public void testValueTypes() { + assertEquals(Tensor.from("tensor<double>(x[1]):{{x:0}:5}").getClass(), IndexedDoubleTensor.class); + assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor<double>(x[1])")).cell(5.0, 0).build().getClass(), + IndexedDoubleTensor.class); + + assertEquals(Tensor.from("tensor<float>(x[1]):{{x:0}:5}").getClass(), IndexedFloatTensor.class); + assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor<float>(x[1])")).cell(5.0, 0).build().getClass(), + IndexedFloatTensor.class); + } + + @Test public void testParseError() { try { Tensor.from("--"); |