diff options
50 files changed, 592 insertions, 341 deletions
diff --git a/application-preprocessor/pom.xml b/application-preprocessor/pom.xml index a021577f838..53d8c6f1849 100644 --- a/application-preprocessor/pom.xml +++ b/application-preprocessor/pom.xml @@ -77,12 +77,10 @@ <finalName>${project.artifactId}-jar-with-dependencies</finalName> <filters> <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> diff --git a/client/go/cmd/clone.go b/client/go/cmd/clone.go index 6fe3c0d5a29..1ba95668d29 100644 --- a/client/go/cmd/clone.go +++ b/client/go/cmd/clone.go @@ -65,41 +65,40 @@ variable.`, }, } -func cloneApplication(source string, name string) { +func cloneApplication(applicationName string, applicationDir string) { zipFile := getSampleAppsZip() defer zipFile.Close() - zipReader, zipOpenError := zip.OpenReader(zipFile.Name()) - if zipOpenError != nil { - fatalErr(zipOpenError, "Could not open sample apps zip '", color.Cyan(zipFile.Name()), "'") + r, err := zip.OpenReader(zipFile.Name()) + if err != nil { + fatalErr(err, "Could not open sample apps zip '", color.Cyan(zipFile.Name()), "'") return } - defer zipReader.Close() + defer r.Close() found := false - for _, f := range zipReader.File { - zipEntryPrefix := "sample-apps-master/" + source + "/" - if strings.HasPrefix(f.Name, zipEntryPrefix) { + for _, f := range r.File { + dirPrefix := "sample-apps-master/" + applicationName + "/" + if strings.HasPrefix(f.Name, dirPrefix) { if !found { // Create destination directory lazily when source is found - createErr := os.Mkdir(name, 0755) + createErr := os.Mkdir(applicationDir, 0755) if createErr != nil { - fatalErr(createErr, "Could not create directory '", color.Cyan(name), "'") + fatalErr(createErr, "Could not create directory '", color.Cyan(applicationDir), "'") return } } found = true - copyError := copy(f, name, zipEntryPrefix) - if copyError != nil { - fatalErr(copyError, "Could not copy zip entry '", color.Cyan(f.Name), "' to ", color.Cyan(name)) + if err := copy(f, applicationDir, dirPrefix); err != nil { + fatalErr(err, "Could not copy zip entry '", color.Cyan(f.Name), "' to ", color.Cyan(applicationDir)) return } } } if !found { - fatalErrHint(fmt.Errorf("Could not find source application '%s'", color.Cyan(source)), "Use -f to ignore the cache") + fatalErrHint(fmt.Errorf("Could not find source application '%s'", color.Cyan(applicationName)), "Use -f to ignore the cache") } else { - log.Print("Created ", color.Cyan(name)) + log.Print("Created ", color.Cyan(applicationDir)) } } @@ -158,6 +157,10 @@ func getSampleAppsZip() *os.File { fatalErr(nil, "Could not download sample apps from GitHub: ", response.StatusCode) return nil } + if err := f.Truncate(0); err != nil { + fatalErr(err, "Could not truncate sample apps file: ", f.Name()) + return nil + } if _, err := io.Copy(f, response.Body); err != nil { fatalErr(err, "Could not write sample apps to file: ", f.Name()) return nil @@ -172,26 +175,25 @@ func copy(f *zip.File, destinationDir string, zipEntryPrefix string) error { destinationPath := filepath.Join(destinationDir, filepath.FromSlash(strings.TrimPrefix(f.Name, zipEntryPrefix))) if strings.HasSuffix(f.Name, "/") { if f.Name != zipEntryPrefix { // root is already created - createError := os.Mkdir(destinationPath, 0755) - if createError != nil { - return createError + if err := os.Mkdir(destinationPath, 0755); err != nil { + return err } } } else { - zipEntry, zipEntryOpenError := f.Open() - if zipEntryOpenError != nil { - return zipEntryOpenError + r, err := f.Open() + if err != nil { + return err } - defer zipEntry.Close() - - destination, createError := os.Create(destinationPath) - if createError != nil { - return createError + defer r.Close() + destination, err := os.Create(destinationPath) + if err != nil { + return err } - - _, copyError := io.Copy(destination, zipEntry) - if copyError != nil { - return copyError + if _, err := io.Copy(destination, r); err != nil { + return err + } + if err := os.Chmod(destinationPath, f.Mode()); err != nil { + return err } } return nil diff --git a/client/go/cmd/clone_test.go b/client/go/cmd/clone_test.go index 054dc7b21fb..af8b686b111 100644 --- a/client/go/cmd/clone_test.go +++ b/client/go/cmd/clone_test.go @@ -15,7 +15,7 @@ import ( ) func TestClone(t *testing.T) { - assertCreated("album-recommendation-selfhosted", "mytestapp", t) + assertCreated("text-search", "mytestapp", t) } func assertCreated(sampleAppName string, app string, t *testing.T) { @@ -32,6 +32,9 @@ func assertCreated(sampleAppName string, app string, t *testing.T) { assert.True(t, util.IsDirectory(filepath.Join(app, "src", "main", "application"))) servicesStat, _ := os.Stat(filepath.Join(app, "src", "main", "application", "services.xml")) - servicesSize := int64(2474) + servicesSize := int64(1772) assert.Equal(t, servicesSize, servicesStat.Size()) + + scriptStat, _ := os.Stat(filepath.Join(app, "bin", "convert-msmarco.sh")) + assert.Equal(t, os.FileMode(0755), scriptStat.Mode()) } diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index f065ae0c680..add81334d35 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -134,11 +134,11 @@ func getService(service string, sessionOrRunID int64, cluster string) *vespa.Ser t := getTarget() timeout := time.Duration(waitSecsArg) * time.Second if timeout > 0 { - log.Printf("Waiting up to %d %s for service to become available ...", color.Cyan(waitSecsArg), color.Cyan("seconds")) + log.Printf("Waiting up to %d %s for %s service to become available ...", color.Cyan(waitSecsArg), color.Cyan("seconds"), color.Cyan(service)) } s, err := t.Service(service, timeout, sessionOrRunID, cluster) if err != nil { - fatalErr(err, "Invalid service: ", service) + fatalErr(err, "Service not found: ", service) } return s } diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go index c686f1d29ad..8480b7a6e2b 100644 --- a/client/go/cmd/prod.go +++ b/client/go/cmd/prod.go @@ -81,6 +81,7 @@ https://cloud.vespa.ai/en/reference/deployment`, "!\nBefore modification a backup of the original file will be created.\n\n") fmt.Fprint(stdout, "A default value is suggested (shown inside brackets) based on\nthe files' existing contents. Press enter to use it.\n\n") fmt.Fprint(stdout, "Abort the configuration at any time by pressing Ctrl-C. The\nfiles will remain untouched.\n\n") + fmt.Fprint(stdout, "See this guide for sizing a Vespa deployment:\n", color.Green("https://docs.vespa.ai/en/performance/sizing-search.html\n\n")) r := bufio.NewReader(stdin) deploymentXML = updateRegions(r, deploymentXML) servicesXML = updateNodes(r, servicesXML) diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index 087be7c352d..023ba18a038 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -31,6 +31,10 @@ Vespa documentation: https://docs.vespa.ai`, PersistentPreRun: func(cmd *cobra.Command, args []string) { configureOutput() }, + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + fatalErr(nil, "invalid command: ", args[0]) + }, } targetArg string diff --git a/client/go/cmd/testdata/sample-apps-master.zip b/client/go/cmd/testdata/sample-apps-master.zip Binary files differindex 6ad49361072..c8fb40af713 100644 --- a/client/go/cmd/testdata/sample-apps-master.zip +++ b/client/go/cmd/testdata/sample-apps-master.zip diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index e3e019f3a76..99da625cf0b 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -31,7 +31,7 @@ const ( queryService = "query" documentService = "document" - waitRetryInterval = 2 * time.Second + retryInterval = 2 * time.Second ) // Service represents a Vespa service. @@ -563,10 +563,10 @@ func wait(fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, time } } timeLeft := deadline.Sub(time.Now()) - if loopOnce || timeLeft < waitRetryInterval { + if loopOnce || timeLeft < retryInterval { break } - time.Sleep(waitRetryInterval) + time.Sleep(retryInterval) } return statusCode, httpErr } diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java index 8cebc091772..246092034de 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/Reindexer.java @@ -209,6 +209,7 @@ public class Reindexer { parameters.setThrottlePolicy(new DynamicThrottlePolicy().setWindowSizeIncrement(speed) .setWindowSizeDecrementFactor(3) .setResizeRate(5) + .setMaxWindowSize(128) .setMinWindowSize(3 + (int) (5 * speed))); parameters.setRemoteDataHandler(cluster.name()); parameters.setMaxPending(8); diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index d946dd972f4..ae9e7a22129 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -397,7 +397,8 @@ ], "fields": [ "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout separate", - "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout leading" + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout leading", + "public static final enum com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout simultaneous" ] }, "com.yahoo.config.application.api.DeploymentSpec": { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 4b019bd9f7a..8ad42b1d4a8 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -564,9 +564,9 @@ public class DeploymentSpec { /** Separate: Application changes wait for upgrade to complete, unless upgrade fails. */ separate, /** Leading: Application changes are allowed to start and catch up to the platform upgrade. */ - leading + leading, // /** Simultaneous: Application changes deploy independently of platform upgrades. */ - // simultaneous + simultaneous } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index b031af9faf2..b12d4024591 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -507,7 +507,7 @@ public class DeploymentSpecXmlReader { switch (rollout) { case "separate": return DeploymentSpec.UpgradeRollout.separate; case "leading": return DeploymentSpec.UpgradeRollout.leading; - // case "simultaneous": return DeploymentSpec.UpgradePolicy.conservative; + case "simultaneous": return DeploymentSpec.UpgradeRollout.simultaneous; default: throw new IllegalArgumentException("Illegal upgrade rollout '" + rollout + "': " + "Must be one of " + Arrays.toString(DeploymentSpec.UpgradeRollout.values())); } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index a97faf5995d..f6af155ffc2 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -386,12 +386,16 @@ public class DeploymentSpecTest { " <instance id='default'>" + " <upgrade rollout='leading' />" + " </instance>" + + " <instance id='aggressive'>" + + " <upgrade rollout='simultaneous' />" + + " </instance>" + " <instance id='custom'/>" + "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); assertEquals("leading", spec.requireInstance("default").upgradeRollout().toString()); assertEquals("separate", spec.requireInstance("custom").upgradeRollout().toString()); + assertEquals("simultaneous", spec.requireInstance("aggressive").upgradeRollout().toString()); } @Test diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml index a855d84de71..2e6634b1f7c 100644 --- a/config-proxy/pom.xml +++ b/config-proxy/pom.xml @@ -93,12 +93,10 @@ <finalName>${project.artifactId}-jar-with-dependencies</finalName> <filters> <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java index 8fd5c8cfdd7..ae9c5285be6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java @@ -185,7 +185,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { if (buildNumber().isEmpty() || o.buildNumber().isEmpty()) return Boolean.compare(buildNumber().isPresent(), o.buildNumber.isPresent()); // Unknown version sorts first - if (deployedDirectly || o.deployedDirectly) + if (deployedDirectly != o.deployedDirectly) return Boolean.compare( ! deployedDirectly, ! o.deployedDirectly); // Directly deployed versions sort first return Long.compare(buildNumber().getAsLong(), o.buildNumber().getAsLong()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 7156e4ac937..a5a1bb93225 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -87,6 +87,7 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.function.Consumer; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -451,10 +452,12 @@ public class ApplicationController { .stream() .flatMap(instance -> instance.deployments().values().stream()) .map(Deployment::applicationVersion) + .filter(version -> !version.isDeployedDirectly()) .sorted() .findFirst() .orElse(ApplicationVersion.unknown); + var olderVersions = application.get().versions() .stream() .filter(version -> version.compareTo(oldestDeployedVersion) < 0) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index ad237a6aa6e..7f7449fb38b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -24,10 +24,13 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -107,10 +110,45 @@ public class DeploymentStatus { return allJobs; } + /** Whether any jobs both dependent on the dependency, and a dependency for the dependent, are failing. */ + private boolean hasFailures(StepStatus dependency, StepStatus dependent) { + Set<StepStatus> dependents = new HashSet<>(); + fillDependents(dependency, new HashSet<>(), dependents, dependent); + Set<JobId> criticalJobs = dependents.stream().flatMap(step -> step.job().stream()).collect(Collectors.toSet()); + + return ! allJobs.matching(job -> criticalJobs.contains(job.id())) + .failing() + .not().outOfTestCapacity() + .isEmpty(); + } + + private boolean fillDependents(StepStatus dependency, Set<StepStatus> visited, Set<StepStatus> dependents, StepStatus current) { + if (visited.contains(current)) + return dependents.contains(current); + + if (dependency == current) + dependents.add(current); + else + for (StepStatus dep : current.dependencies) + if (fillDependents(dependency, visited, dependents, dep)) + dependents.add(current); + + visited.add(current); + return dependents.contains(current); + } + + /** Whether any job is failing on anything older than version, with errors other than lack of capacity in a test zone.. */ + public boolean hasFailures(ApplicationVersion version) { + return ! allJobs.failing() + .not().outOfTestCapacity() + .matching(job -> job.lastTriggered().get().versions().targetApplication().compareTo(version) < 0) + .isEmpty(); + } + /** Whether any jobs of this application are failing with other errors than lack of capacity in a test zone. */ public boolean hasFailures() { return ! allJobs.failing() - .not().withStatus(RunStatus.outOfCapacity) + .not().outOfTestCapacity() .isEmpty(); } @@ -129,13 +167,13 @@ public class DeploymentStatus { /** * The set of jobs that need to run for the changes of each instance of the application to be considered complete, - * and any test jobs for any oustanding change, which will likely be needed to lated deploy this change. + * and any test jobs for any outstanding change, which will likely be needed to later deploy this change. */ - public Map<JobId, List<Versions>> jobsToRun() { + public Map<JobId, List<Job>> jobsToRun() { Map<InstanceName, Change> changes = new LinkedHashMap<>(); for (InstanceName instance : application.deploymentSpec().instanceNames()) changes.put(instance, application.require(instance).change()); - Map<JobId, List<Versions>> jobs = jobsToRun(changes); + Map<JobId, List<Job>> jobs = jobsToRun(changes); // Add test jobs for any outstanding change. for (InstanceName instance : application.deploymentSpec().instanceNames()) @@ -151,12 +189,12 @@ public class DeploymentStatus { Collections::unmodifiableMap)); } - private Map<JobId, List<Versions>> jobsToRun(Map<InstanceName, Change> changes, boolean eagerTests) { - Map<JobId, Versions> productionJobs = new LinkedHashMap<>(); + private Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes, boolean eagerTests) { + Map<JobId, List<Job>> productionJobs = new LinkedHashMap<>(); changes.forEach((instance, change) -> productionJobs.putAll(productionJobs(instance, change, eagerTests))); - Map<JobId, List<Versions>> testJobs = testJobs(productionJobs); - Map<JobId, List<Versions>> jobs = new LinkedHashMap<>(testJobs); - productionJobs.forEach((job, versions) -> jobs.put(job, List.of(versions))); + Map<JobId, List<Job>> testJobs = testJobs(productionJobs); + Map<JobId, List<Job>> jobs = new LinkedHashMap<>(testJobs); + jobs.putAll(productionJobs); // Add runs for idle, declared test jobs if they have no successes on their instance's change's versions. jobSteps.forEach((job, step) -> { if ( ! step.isDeclared() || jobs.containsKey(job)) @@ -173,13 +211,13 @@ public class DeploymentStatus { Versions versions = Versions.from(change, application, firstProductionJobWithDeployment.flatMap(this::deploymentFor), systemVersion); if (step.completedAt(change, firstProductionJobWithDeployment).isEmpty()) - jobs.merge(job, List.of(versions), DeploymentStatus::union); + jobs.merge(job, List.of(new Job(versions, step.readyAt(change), change)), DeploymentStatus::union); }); return Collections.unmodifiableMap(jobs); } /** The set of jobs that need to run for the given changes to be considered complete. */ - public Map<JobId, List<Versions>> jobsToRun(Map<InstanceName, Change> changes) { + public Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes) { return jobsToRun(changes, false); } @@ -239,61 +277,151 @@ public class DeploymentStatus { .distinct(); } - /** - * True if the job has already been triggered on the given versions, or if all test types (systemTest, stagingTest), - * restricted to the job's instance if declared in that instance, have successful runs on the given versions. - */ - public boolean isTested(JobId job, Change change) { - Versions versions = Versions.from(change, application, deploymentFor(job), systemVersion); - return allJobs.triggeredOn(versions).get(job).isPresent() - || Stream.of(systemTest, stagingTest) - .noneMatch(testType -> declaredTest(job.application(), testType).map(__ -> allJobs.instance(job.application().instance())) - .orElse(allJobs) - .type(testType) - .successOn(versions).isEmpty()); - } - - private Map<JobId, Versions> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) { - Map<JobId, Versions> jobs = new LinkedHashMap<>(); + /** Earliest instant when job was triggered with given versions, or both system and staging tests were successful. */ + public Optional<Instant> verifiedAt(JobId job, Versions versions) { + Optional<Instant> triggeredAt = allJobs.get(job) + .flatMap(status -> status.runs().values().stream() + .filter(run -> run.versions().equals(versions)) + .findFirst()) + .map(Run::start); + Optional<Instant> systemTestedAt = testedAt(job.application(), systemTest, versions); + Optional<Instant> stagingTestedAt = testedAt(job.application(), stagingTest, versions); + if (systemTestedAt.isEmpty() || stagingTestedAt.isEmpty()) return triggeredAt; + Optional<Instant> testedAt = systemTestedAt.get().isAfter(stagingTestedAt.get()) ? systemTestedAt : stagingTestedAt; + return triggeredAt.isPresent() && triggeredAt.get().isBefore(testedAt.get()) ? triggeredAt : testedAt; + } + + /** Earliest instant when versions were tested for the given instance */ + private Optional<Instant> testedAt(ApplicationId instance, JobType type, Versions versions) { + return declaredTest(instance, type).map(__ -> allJobs.instance(instance.instance())) + .orElse(allJobs) + .type(type).asList().stream() + .flatMap(status -> RunList.from(status) + .on(versions) + .status(RunStatus.success) + .asList().stream() + .map(Run::start)) + .min(naturalOrder()); + } + + private Map<JobId, List<Job>> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) { + Map<JobId, List<Job>> jobs = new LinkedHashMap<>(); jobSteps.forEach((job, step) -> { // When computing eager test jobs for outstanding changes, assume current upgrade completes successfully. Optional<Deployment> deployment = deploymentFor(job) - .map(existing -> assumeUpgradesSucceed ? new Deployment(existing.zone(), - existing.applicationVersion(), - change.platform().orElse(existing.version()), - existing.at(), - existing.metrics(), - existing.activity(), - existing.quota(), - existing.cost()) - : existing); - if ( job.application().instance().equals(instance) - && job.type().isProduction() - && step.completedAt(change, Optional.of(job)).isEmpty()) // Signal strict completion criterion by depending on job itself. - jobs.put(job, Versions.from(change, application, deployment, systemVersion)); + .map(existing -> assumeUpgradesSucceed ? withChange(existing, change.withoutApplication()) : existing); + if (job.application().instance().equals(instance) && job.type().isProduction()) { + + List<Job> toRun = new ArrayList<>(); + List<Change> changes = changes(job, step, change, deployment); + if (changes.isEmpty()) return; + for (Change partial : changes) { + toRun.add(new Job(Versions.from(partial, application, deployment, systemVersion), + step.readyAt(partial, Optional.of(job)), + partial)); + // Assume first partial change is applied before the second. + deployment = deployment.map(existing -> withChange(existing, partial)); + } + jobs.put(job, toRun); + } }); return jobs; } + private static Deployment withChange(Deployment deployment, Change change) { + return new Deployment(deployment.zone(), + change.application().orElse(deployment.applicationVersion()), + change.platform().orElse(deployment.version()), + deployment.at(), + deployment.metrics(), + deployment.activity(), + deployment.quota(), + deployment.cost()); + } + + /** Changes to deploy with the given job, possibly split in two steps. */ + private List<Change> changes(JobId job, StepStatus step, Change change, Optional<Deployment> deployment) { + // Signal strict completion criterion by depending on job itself. + if (step.completedAt(change, Optional.of(job)).isPresent()) + return List.of(); + + if ( step.completedAt(change.withoutApplication(), Optional.of(job)).isPresent() + || step.completedAt(change.withoutPlatform(), Optional.of(job)).isPresent()) + return List.of(change); + + if (change.platform().isEmpty() || change.application().isEmpty() || change.isPinned()) + return List.of(change); + + Optional<Instant> platformReadyAt = step.dependenciesCompletedAt(change.withoutApplication(), Optional.of(job)); + Optional<Instant> revisionReadyAt = step.dependenciesCompletedAt(change.withoutPlatform(), Optional.of(job)); + + // If the revision is not ready for this job, we prioritise the upgrade, assuming it was triggered first, + // unless this is a production test job, and the upgrade is already failing between deployment and here, + // in which case we must abandon the production test on the pure upgrade, so the revision can be deployed. + if (revisionReadyAt.isEmpty()) { + if (isTestAndIsFailingBetweenDeploymentAndThis(job)) return List.of(change); + return List.of(change.withoutApplication(), change); + } + // If only the revision is ready, we prioritise that. + if (platformReadyAt.isEmpty()) return List.of(change.withoutPlatform(), change); + + // Both changes are ready for this step, so we use the policy to decide. + // TODO jonmv adding change.application.at and change.platform.at makes this better + boolean platformReadyFirst = platformReadyAt.get().isBefore(revisionReadyAt.get()); + boolean revisionReadyFirst = revisionReadyAt.get().isBefore(platformReadyAt.get()); + switch (application.deploymentSpec().requireInstance(job.application().instance()).upgradeRollout()) { + case separate: // Let whichever change rolled out first, keep rolling first, unless upgrade alone is failing. + return (platformReadyFirst || platformReadyAt.get().equals(Instant.EPOCH)) // Assume platform was first if no jobs have run yet. + ? step.job().flatMap(jobs()::get).flatMap(JobStatus::firstFailing).isPresent() + ? List.of(change) // Platform was first, but is failing. + : List.of(change.withoutApplication(), change) // Platform was first, and is OK. + : revisionReadyFirst + ? List.of(change.withoutPlatform(), change) // Revision was first. + : List.of(change); // Both ready at the same time, probably due to earlier failure. + case leading: // When one change catches up, they fuse and continue together. + return List.of(change); + case simultaneous: // Revisions are allowed to run ahead, but the job where it caught up should have both changes. + return platformReadyFirst ? List.of(change) : List.of(change.withoutPlatform(), change); + default: throw new IllegalStateException("Unknown upgrade rollout policy"); + } + } + + private boolean isTestAndIsFailingBetweenDeploymentAndThis(JobId job) { + if ( ! job.type().isTest()) return false; + JobId deployment = new JobId(job.application(), JobType.from(system, job.type().zone(system)).get()); + return hasFailures(jobSteps.get(deployment), jobSteps.get(job)); + } + /** The test jobs that need to run prior to the given production deployment jobs. */ - public Map<JobId, List<Versions>> testJobs(Map<JobId, Versions> jobs) { - Map<JobId, List<Versions>> testJobs = new LinkedHashMap<>(); + public Map<JobId, List<Job>> testJobs(Map<JobId, List<Job>> jobs) { + Map<JobId, List<Job>> testJobs = new LinkedHashMap<>(); for (JobType testType : List.of(systemTest, stagingTest)) { - jobs.forEach((job, versions) -> { + jobs.forEach((job, versionsList) -> { if (job.type().isProduction() && job.type().isDeployment()) { declaredTest(job.application(), testType).ifPresent(testJob -> { - if (allJobs.successOn(versions).get(testJob).isEmpty()) - testJobs.merge(testJob, List.of(versions), DeploymentStatus::union); + for (Job productionJob : versionsList) + if (allJobs.successOn(productionJob.versions()).get(testJob).isEmpty()) + testJobs.merge(testJob, List.of(new Job(productionJob.versions(), + jobSteps().get(testJob).readyAt(productionJob.change), + productionJob.change)), + DeploymentStatus::union); }); } }); - jobs.forEach((job, versions) -> { - if ( job.type().isProduction() && job.type().isDeployment() - && allJobs.successOn(versions).type(testType).isEmpty() - && testJobs.keySet().stream() - .noneMatch(test -> test.type() == testType - && testJobs.get(test).contains(versions))) - testJobs.merge(firstDeclaredOrElseImplicitTest(testType), List.of(versions), DeploymentStatus::union); + jobs.forEach((job, versionsList) -> { + for (Job productionJob : versionsList) + if ( job.type().isProduction() && job.type().isDeployment() + && allJobs.successOn(productionJob.versions()).type(testType).isEmpty() + && testJobs.keySet().stream() + .noneMatch(test -> test.type() == testType + && testJobs.get(test).stream().anyMatch(testJob -> testJob.versions().equals(productionJob.versions())))) { + JobId testJob = firstDeclaredOrElseImplicitTest(testType); + testJobs.merge(testJob, + List.of(new Job(productionJob.versions(), + jobSteps.get(testJob).readyAt(productionJob.change), + productionJob.change)), + DeploymentStatus::union); + } }); } return Collections.unmodifiableMap(testJobs); @@ -535,11 +663,10 @@ public class DeploymentStatus { Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { return ( (change.platform().isEmpty() || change.platform().equals(instance.change().platform())) && (change.application().isEmpty() || change.application().equals(instance.change().application())) - || status.jobsToRun(Map.of(instance.name(), change)).isEmpty()) + || step().steps().stream().noneMatch(step -> step.concerns(prod))) ? dependenciesCompletedAt(change, dependent) : Optional.empty(); } - // TODO jonmv: complete for p-jobs: last is XXX, but ready/verified uses any is XXX. @Override public Optional<Instant> blockedUntil(Change change) { @@ -614,8 +741,10 @@ public class DeploymentStatus { @Override public Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { - return super.readyAt(change, Optional.of(job.id())) - .filter(__ -> status.isTested(job.id(), change)); + Optional<Instant> readyAt = super.readyAt(change, dependent); + Optional<Instant> testedAt = status.verifiedAt(job.id(), Versions.from(change, status.application, existingDeployment, status.systemVersion)); + if (readyAt.isEmpty() || testedAt.isEmpty()) return Optional.empty(); + return readyAt.get().isAfter(testedAt.get()) ? readyAt : testedAt; } /** Complete if deployment is on pinned version, and last successful deployment, or if given versions is strictly a downgrade, and this isn't forced by a pin. */ @@ -641,8 +770,9 @@ public class DeploymentStatus { : RunList.from(job).status(RunStatus.success).asList().stream()) .filter(run -> change.platform().map(run.versions().targetPlatform()::equals).orElse(true) && change.application().map(run.versions().targetApplication()::equals).orElse(true)) - .findFirst() - .flatMap(Run::end); + .map(Run::end) + .flatMap(Optional::stream) + .min(naturalOrder()); } }; } @@ -696,4 +826,39 @@ public class DeploymentStatus { } + public static class Job { + + private final Versions versions; + private final Optional<Instant> readyAt; + private final Change change; + + public Job(Versions versions, Optional<Instant> readyAt, Change change) { + this.versions = versions; + this.readyAt = readyAt; + this.change = change; + } + + public Versions versions() { + return versions; + } + + public Optional<Instant> readyAt() { + return readyAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Job job = (Job) o; + return versions.equals(job.versions) && readyAt.equals(job.readyAt) && change.equals(job.change); + } + + @Override + public int hashCode() { + return Objects.hash(versions, readyAt, change); + } + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 9790ece421e..b944593bbab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -97,7 +97,7 @@ public class DeploymentTrigger { && status.instanceSteps().get(instanceName) .readyAt(outstanding) .map(readyAt -> ! readyAt.isAfter(clock.instant())).orElse(false) - && acceptNewApplicationVersion(status, instanceName)) { + && acceptNewApplicationVersion(status, instanceName, outstanding.application().get())) { application = application.with(instanceName, instance -> withRemainingChange(instance, instance.change().with(outstanding.application().get()), status)); } @@ -198,11 +198,12 @@ public class DeploymentTrigger { DeploymentStatus status = jobs.deploymentStatus(application); Versions versions = Versions.from(instance.change(), application, status.deploymentFor(job), controller.readSystemVersion()); - Map<JobId, List<Versions>> jobs = status.testJobs(Map.of(job, versions)); + DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(versions, Optional.of(controller.clock().instant()), instance.change()); + Map<JobId, List<DeploymentStatus.Job>> jobs = status.testJobs(Map.of(job, List.of(toTrigger))); if (jobs.isEmpty() || ! requireTests) - jobs = Map.of(job, List.of(versions)); + jobs = Map.of(job, List.of(toTrigger)); jobs.forEach((jobId, versionsList) -> { - trigger(deploymentJob(instance, versionsList.get(0), jobId.type(), status.jobs().get(jobId).get(), clock.instant())); + trigger(deploymentJob(instance, versionsList.get(0).versions(), jobId.type(), status.jobs().get(jobId).get(), clock.instant())); }); return List.copyOf(jobs.keySet()); } @@ -321,19 +322,18 @@ public class DeploymentTrigger { /** Finds the next step to trigger for the given application, if any, and returns these as a list. */ private List<Job> computeReadyJobs(DeploymentStatus status) { List<Job> jobs = new ArrayList<>(); - status.jobsToRun().forEach((job, versionsList) -> { - for (Versions versions : versionsList) - status.jobSteps().get(job).readyAt(status.application().require(job.application().instance()).change()) - .filter(readyAt -> ! clock.instant().isBefore(readyAt)) - .filter(__ -> ! (job.type().isProduction() && isUnhealthyInAnotherZone(status.application(), job))) - .filter(__ -> abortIfRunning(versionsList, status.jobs().get(job).get())) // Abort and trigger this later if running with outdated parameters. - .ifPresent(readyAt -> { - jobs.add(deploymentJob(status.application().require(job.application().instance()), - versions, - job.type(), - status.instanceJobs(job.application().instance()).get(job.type()), - readyAt)); - }); + Map<JobId, List<DeploymentStatus.Job>> jobsToRun = status.jobsToRun(); + jobsToRun.forEach((job, versionsList) -> { + versionsList.get(0).readyAt() + .filter(readyAt -> ! clock.instant().isBefore(readyAt)) + .filter(__ -> ! (job.type().isProduction() && isUnhealthyInAnotherZone(status.application(), job))) + .filter(__ -> abortIfRunning(status, jobsToRun, job)) // Abort and trigger this later if running with outdated parameters. + .map(readyAt -> deploymentJob(status.application().require(job.application().instance()), + versionsList.get(0).versions(), + job.type(), + status.instanceJobs(job.application().instance()).get(job.type()), + readyAt)) + .ifPresent(jobs::add); }); return Collections.unmodifiableList(jobs); } @@ -348,29 +348,50 @@ public class DeploymentTrigger { return false; } - /** Returns whether the job is not running, and also aborts it if it's running with outdated versions. */ - private boolean abortIfRunning(List<Versions> versionsList, JobStatus status) { - if ( ! status.isRunning()) - return true; + private void abortIfOutdated(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { + status.jobs().get(job) + .flatMap(JobStatus::lastTriggered) + .filter(last -> ! last.hasEnded()) + .ifPresent(last -> { + if (jobs.get(job).stream().noneMatch(versions -> versions.versions().targetsMatch(last.versions()) + && versions.versions().sourcesMatchIfPresent(last.versions()))) { + log.log(Level.INFO, "Aborting outdated run " + last); + controller.jobController().abort(last.id()); + } + }); + } - Run last = status.lastTriggered().get(); - if (versionsList.stream().noneMatch(versions -> versions.targetsMatch(last.versions()) - && versions.sourcesMatchIfPresent(last.versions()))) - controller.jobController().abort(last.id()); + /** Returns whether the job is free to start, and also aborts it if it's running with outdated versions. */ + private boolean abortIfRunning(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { + abortIfOutdated(status, jobs, job); + boolean blocked = status.jobs().get(job).get().isRunning(); + + if ( ! job.type().isTest()) { + Optional<JobStatus> productionTest = JobType.testFrom(controller.system(), job.type().zone(controller.system()).region()) + .map(type -> new JobId(job.application(), type)) + .flatMap(status.jobs()::get); + if (productionTest.isPresent()) { + abortIfOutdated(status, jobs, productionTest.get().id()); + // Production deployments are also blocked by their declared tests, if the next versions to run + // for those are not the same as the versions we're considering running in the deployment job now. + if (productionTest.map(JobStatus::id).map(jobs::get) + .map(versions -> ! versions.get(0).versions().targetsMatch(jobs.get(job).get(0).versions())) + .orElse(false)) + blocked = true; + } + } - return false; + return ! blocked; } // ---------- Change management o_O ---------- - private boolean acceptNewApplicationVersion(DeploymentStatus status, InstanceName instance) { + private boolean acceptNewApplicationVersion(DeploymentStatus status, InstanceName instance, ApplicationVersion version) { if (status.application().deploymentSpec().instance(instance).isEmpty()) return false; // Unknown instance. - if (status.hasFailures()) return true; // Allow changes to fix upgrade or previous revision problems. + if (status.hasFailures(version)) return true; // Allow changes to fix upgrade or previous revision problems. DeploymentInstanceSpec spec = status.application().deploymentSpec().requireInstance(instance); Change change = status.application().require(instance).change(); - if (change.platform().isPresent() && spec.upgradeRollout() == DeploymentSpec.UpgradeRollout.separate) return false; - if (change.application().isPresent() && spec.upgradeRevision() == DeploymentSpec.UpgradeRevision.separate) return false; - return true; + return change.application().isEmpty() || spec.upgradeRevision() != DeploymentSpec.UpgradeRevision.separate; } private Instance withRemainingChange(Instance instance, Change change, DeploymentStatus status) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java index 5021d6b2076..fe9c6e6f3d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -12,6 +12,7 @@ import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; @@ -51,6 +52,10 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { return matching(job -> job.lastCompleted().isPresent() && ! job.isSuccess()); } + public JobList outOfTestCapacity() { + return matching(job -> job.isOutOfCapacity() && job.id().type().environment().isTest()); + } + public JobList running() { return matching(job -> job.isRunning()); } @@ -76,8 +81,13 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { } /** Returns the subset of jobs run for the given instance. */ - public JobList instance(InstanceName instance) { - return matching(job -> job.id().application().instance().equals(instance)); + public JobList instance(InstanceName... instances) { + return instance(Set.of(instances)); + } + + /** Returns the subset of jobs run for the given instance. */ + public JobList instance(Collection<InstanceName> instances) { + return matching(job -> instances.contains(job.id().application().instance())); } /** Returns the subset of jobs of which are production jobs. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java index 925bd500199..00cd4bd5c6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunList.java @@ -26,7 +26,7 @@ public class RunList extends AbstractFilteringList<Run, RunList> { return from(job.runs().descendingMap().values()); } - /** Returns the jobs with runs matching the given versions — targets only for system test, everything present otherwise. */ + /** Returns the jobs with runs matching the given versions — targets only for system test, everything present otherwise. */ public RunList on(Versions versions) { return matching(run -> matchingVersions(run, versions)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 4a3368a7228..7377f9ab1e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -229,13 +229,15 @@ public class ApplicationSerializer { } private void versionsToSlime(Application application, Cursor object) { - // Add all deployed versions as well + // Add all indirectly deployed versions as well + // Remove after all apps are updated application.instances() .values() .stream() .flatMap(instance -> instance.deployments().values().stream()) .map(Deployment::applicationVersion) .forEach(application.versions()::add); + application.versions().removeIf(ApplicationVersion::isDeployedDirectly); application.versions().forEach(version -> toSlime(version, object.addObject())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 30bd040a682..3244779abb7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -256,7 +256,7 @@ class JobControllerApiHandlerHelper { responseObject.setString("application", id.application().value()); application.projectId().ifPresent(projectId -> responseObject.setLong("projectId", projectId)); - Map<JobId, List<Versions>> jobsToRun = status.jobsToRun(); + Map<JobId, List<DeploymentStatus.Job>> jobsToRun = status.jobsToRun(); Cursor stepsArray = responseObject.setArray("steps"); VersionStatus versionStatus = controller.readVersionStatus(); for (DeploymentStatus.StepStatus stepStatus : status.allSteps()) { @@ -330,17 +330,17 @@ class JobControllerApiHandlerHelper { JobStatus jobStatus = status.jobs().get(job).get(); Cursor toRunArray = stepObject.setArray("toRun"); - for (Versions versions : jobsToRun.getOrDefault(job, List.of())) { + for (DeploymentStatus.Job versions : jobsToRun.getOrDefault(job, List.of())) { boolean running = jobStatus.lastTriggered() .map(run -> jobStatus.isRunning() - && versions.targetsMatch(run.versions()) - && (job.type().isProduction() || versions.sourcesMatchIfPresent(run.versions()))) + && versions.versions().targetsMatch(run.versions()) + && (job.type().isProduction() || versions.versions().sourcesMatchIfPresent(run.versions()))) .orElse(false); if (running) continue; // Run will be contained in the "runs" array. Cursor runObject = toRunArray.addObject(); - toSlime(runObject.setObject("versions"), versions); + toSlime(runObject.setObject("versions"), versions.versions()); } toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), 10, baseUriForJob); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 5a6d929b054..158cc6caede 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -185,6 +185,7 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { .ifPresent(until -> jobObject.setLong("coolingDownUntil", until.toEpochMilli())); if (jobsToRun.containsKey(job)) { List<Versions> versionsOnThisPlatform = jobsToRun.get(job).stream() + .map(DeploymentStatus.Job::versions) .filter(versions -> versions.targetPlatform().equals(statistics.version())) .collect(Collectors.toList()); if ( ! versionsOnThisPlatform.isEmpty()) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 699721b128c..9c5f7e3376a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -164,14 +164,13 @@ public class DeploymentContext { } - /** Completely deploy the latest change */ + /** Completely deploy the current change */ public DeploymentContext deploy() { Application application = application(); assertTrue("Application package submitted", application.latestVersion().isPresent()); assertFalse("Submission is not already deployed", application.instances().values().stream() .anyMatch(instance -> instance.deployments().values().stream() .anyMatch(deployment -> deployment.applicationVersion().equals(lastSubmission)))); - assertEquals(application.latestVersion(), instance().change().application()); completeRollout(application.deploymentSpec().instances().size() > 1); for (var instance : application().instances().values()) { assertFalse(instance.change().hasTargets()); @@ -334,20 +333,20 @@ public class DeploymentContext { /** Runs and returns all remaining jobs for the application, at most once, and asserts the current change is rolled out. */ public DeploymentContext completeRollout(boolean multiInstance) { triggerJobs(); - Map<ApplicationId, Set<JobType>> jobsByInstance = new HashMap<>(); + Map<ApplicationId, Map<JobType, Versions>> jobsByInstance = new HashMap<>(); List<Run> activeRuns; while ( ! (activeRuns = this.jobs.active(applicationId)).isEmpty()) for (Run run : activeRuns) { - Set<JobType> jobs = jobsByInstance.computeIfAbsent(run.id().application(), k -> new HashSet<>()); - if (jobs.add(run.id().type())) { - runJob(run.id().type(), run.id().application()); - if (multiInstance) { - tester.outstandingChangeDeployer().run(); - } - triggerJobs(); - } else { - throw new AssertionError("Job '" + run.id() + "' was run twice"); + Map<JobType, Versions> jobs = jobsByInstance.computeIfAbsent(run.id().application(), k -> new HashMap<>()); + Versions previous = jobs.put(run.id().type(), run.versions()); + if (run.versions().equals(previous)) { + throw new AssertionError("Job '" + run.id() + "' was run twice on same versions"); } + runJob(run.id().type(), run.id().application()); + if (multiInstance) { + tester.outstandingChangeDeployer().run(); + } + triggerJobs(); } assertFalse("Change should have no targets, but was " + instance().change(), instance().change().hasTargets()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index cd1e12312a8..0ac1691829a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -189,9 +189,10 @@ public class DeploymentTriggerTest { tester.controllerTester().upgradeSystem(new Version("8.9")); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest); + tester.clock().advance(Duration.ofMinutes(1)); tester.triggerJobs(); - // Jobs are not aborted when the new submission remains outstanding. + // Upgrade is allowed to proceed ahead of revision change, and is not aborted. app.submit(); app.runJob(systemTest).runJob(stagingTest); tester.triggerJobs(); @@ -381,17 +382,15 @@ public class DeploymentTriggerTest { // Application on (6.1, 1.0.1) Version v1 = Version.fromString("6.1"); - // Application is mid-upgrade when block window begins, and has an outstanding change. + // Application is mid-upgrade when block window begins, and gets an outstanding change. Version v2 = Version.fromString("6.2"); tester.controllerTester().upgradeSystem(v2); tester.upgrader().maintain(); - app.submit(applicationPackage); - app.runJob(stagingTest).runJob(systemTest); // Entering block window will keep the outstanding change in place. tester.clock().advance(Duration.ofHours(1)); - tester.outstandingChangeDeployer().run(); + app.submit(applicationPackage); app.runJob(productionUsWest1); assertEquals(1, app.instanceJobs().get(productionUsWest1).lastSuccess().get().versions().targetApplication().buildNumber().getAsLong()); assertEquals(2, app.deploymentStatus().outstandingChange(app.instance().name()).application().get().buildNumber().getAsLong()); @@ -599,19 +598,23 @@ public class DeploymentTriggerTest { // Change has a higher application version than what is deployed -- deployment should trigger. app1.timeOutUpgrade(productionUsCentral1); - assertEquals(version2, app1.instance().deployments().get(productionUsCentral1.zone(main)).version()); + assertEquals(version2, app1.deployment(productionUsCentral1.zone(main)).version()); assertEquals(revision2, app1.deployment(productionUsCentral1.zone(main)).applicationVersion()); // Change is again strictly dominated, and us-central-1 is skipped, even though it is still failing. - tester.clock().advance(Duration.ofHours(2).plus(Duration.ofSeconds(1))); // Enough time for retry + tester.clock().advance(Duration.ofHours(3)); // Enough time for retry tester.triggerJobs(); // Failing job is not retried as change has been deployed app1.assertNotRunning(productionUsCentral1); // Last job has a different deployment target, so tests need to run again. - app1.runJob(systemTest).runJob(stagingTest).runJob(productionEuWest1); - assertFalse(app1.instance().change().hasTargets()); - assertFalse(app1.instanceJobs().get(productionUsCentral1).isSuccess()); + app1.runJob(systemTest) + .runJob(stagingTest) // Eager test of outstanding change, assuming upgrade in west succeeds. + .runJob(productionEuWest1) // Upgrade completes, and revision is the only change. + .runJob(productionUsCentral1) // With only revision change, central should run to cover a previous failure. + .runJob(productionEuWest1); // Finally, west changes revision. + assertEquals(Change.empty(), app1.instance().change()); + assertEquals(Optional.of(RunStatus.success), app1.instanceJobs().get(productionUsCentral1).lastStatus()); } @Test @@ -1168,48 +1171,65 @@ public class DeploymentTriggerTest { assertEquals(Change.empty(), app2.instance().change()); assertEquals(Change.empty(), app3.instance().change()); - // Upgrade instance 1; a failure in any instance allows an application change to accompany the upgrade. + // Upgrade instance 1; upgrade rolls out first, with revision following. // The new platform won't roll out to the conservative instance until the normal one is upgraded. - app2.failDeployment(systemTest); app1.submit(applicationPackage); assertEquals(Change.of(version).with(app1.application().latestVersion().get()), app1.instance().change()); + // Upgrade platform. app2.runJob(systemTest); - app1.jobAborted(stagingTest) - .runJob(stagingTest) + app1.runJob(stagingTest) .runJob(productionUsWest1) .runJob(productionUsEast3); - app1.runJob(stagingTest); // Tests with only the outstanding application change. - app2.runJob(systemTest); // Tests with only the outstanding application change. + // Upgrade revision + tester.clock().advance(Duration.ofSeconds(1)); // Ensure we see revision as rolling after upgrade. + app2.runJob(systemTest); // R + app1.runJob(stagingTest) // R + .runJob(productionUsWest1); // R + // productionUsEast3 won't change revision before its production test has completed for the upgrade, which is one of the last jobs! tester.clock().advance(Duration.ofHours(2)); app1.runJob(productionEuWest1); tester.clock().advance(Duration.ofHours(1)); app1.runJob(productionAwsUsEast1a); - tester.triggerJobs(); app1.runJob(testAwsUsEast1a); + tester.clock().advance(Duration.ofSeconds(1)); + app1.runJob(productionAwsUsEast1a); // R + app1.runJob(testAwsUsEast1a); // R app1.runJob(productionApNortheast2); app1.runJob(productionApNortheast1); tester.clock().advance(Duration.ofHours(1)); app1.runJob(testApNortheast1); app1.runJob(testApNortheast2); + app1.runJob(productionApNortheast2); // R + app1.runJob(productionApNortheast1); // R app1.runJob(testUsEast3); app1.runJob(productionApSoutheast1); + tester.clock().advance(Duration.ofSeconds(1)); + app1.runJob(productionUsEast3); // R + tester.clock().advance(Duration.ofHours(2)); + app1.runJob(productionEuWest1); // R + tester.clock().advance(Duration.ofMinutes(330)); + app1.runJob(testApNortheast1); // R + app1.runJob(testApNortheast2); // R + app1.runJob(testUsEast3); // R + app1.runJob(productionApSoutheast1); // R + + app1.runJob(stagingTest); // Tests with only the outstanding application change. + app2.runJob(systemTest); // Tests with only the outstanding application change. // Confidence rises to high, for the new version, and instance 2 starts to upgrade. tester.controllerTester().computeVersionStatus(); tester.upgrader().maintain(); tester.outstandingChangeDeployer().run(); tester.triggerJobs(); - assertEquals(2, tester.jobs().active().size()); + assertEquals(tester.jobs().active().toString(), 1, tester.jobs().active().size()); assertEquals(Change.empty(), app1.instance().change()); assertEquals(Change.of(version), app2.instance().change()); assertEquals(Change.empty(), app3.instance().change()); - app1.runJob(stagingTest); // Never completed successfully with just the upgrade. - app2.runJob(systemTest) // Never completed successfully with just the upgrade. - .runJob(productionEuWest1) + app2.runJob(productionEuWest1) .failDeployment(testEuWest1); - // Instance 2 failed the last job, and now exist block window, letting application change roll out with the upgrade. + // Instance 2 failed the last job, and now exits block window, letting application change roll out with the upgrade. tester.clock().advance(Duration.ofDays(1)); // Leave block window for revisions. tester.upgrader().maintain(); tester.outstandingChangeDeployer().run(); @@ -1224,41 +1244,48 @@ public class DeploymentTriggerTest { assertEquals(Change.empty(), app2.instance().change()); assertEquals(Change.empty(), app3.instance().change()); - // Two first instances upgraded and with new revision — last instance gets change from whatever maintainer runs first. + // Two first instances upgraded and with new revision — last instance gets both changes as well. tester.upgrader().maintain(); tester.outstandingChangeDeployer().run(); - assertEquals(Change.of(version), app3.instance().change()); + assertEquals(Change.of(version).with(app1.lastSubmission().get()), app3.instance().change()); tester.deploymentTrigger().cancelChange(app3.instanceId(), ALL); tester.outstandingChangeDeployer().run(); tester.upgrader().maintain(); - assertEquals(Change.of(app1.application().latestVersion().get()), app3.instance().change()); + assertEquals(Change.of(app1.lastSubmission().get()), app3.instance().change()); app3.runJob(productionEuWest1); tester.upgrader().maintain(); + app1.runJob(stagingTest); app3.runJob(productionEuWest1); tester.triggerJobs(); assertEquals(List.of(), tester.jobs().active()); assertEquals(Change.empty(), app3.instance().change()); } + // TODO test new revision allowed exactly when dependent instance is failing + // TODO test multi-tier pipeline with various policies + // TODO test multi-tier pipeline with upgrade after new version is the candidate + @Test - public void testChangeCompletion() { + public void testRevisionJoinsUpgradeWithSeparateRollout() { var app = tester.newDeploymentContext().submit().deploy(); var version = new Version("7.1"); tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + tester.clock().advance(Duration.ofMinutes(1)); app.submit(); - tester.triggerJobs(); - tester.outstandingChangeDeployer().run(); - assertEquals(Change.of(version), app.instance().change()); + assertEquals(Change.of(version).with(app.lastSubmission().get()), app.instance().change()); + app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); app.runJob(productionUsEast3).runJob(productionUsWest1); tester.triggerJobs(); - tester.outstandingChangeDeployer().run(); assertEquals(Change.of(app.lastSubmission().get()), app.instance().change()); + + app.runJob(productionUsEast3).runJob(productionUsWest1); + assertEquals(Change.empty(), app.instance().change()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 8e90cfc69f3..acaa8b24d9d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; @@ -30,42 +31,34 @@ public class OutstandingChangeDeployerTest { OutstandingChangeDeployer deployer = new OutstandingChangeDeployer(tester.controller(), Duration.ofMinutes(10)); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-west-1") + .upgradeRevision("separate") .build(); - var app1 = tester.newDeploymentContext("tenant", "app1", "default").submit(applicationPackage).deploy(); + var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); Version version = new Version(6, 2); - tester.deploymentTrigger().triggerChange(app1.instanceId(), Change.of(version)); - tester.deploymentTrigger().triggerReadyJobs(); + tester.deploymentTrigger().triggerChange(app.instanceId(), Change.of(version)); + assertEquals(Change.of(version), app.instance().change()); + assertFalse(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); - assertEquals(Change.of(version), app1.instance().change()); - assertFalse(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); + app.submit(applicationPackage); + Optional<ApplicationVersion> revision = app.lastSubmission(); + assertFalse(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); + assertEquals(Change.of(version).with(revision.get()), app.instance().change()); - assertEquals(1, app1.application().latestVersion().get().buildNumber().getAsLong()); - app1.submit(applicationPackage, Optional.of(new SourceRevision("repository1", "master", "cafed00d"))); + app.submit(applicationPackage); + Optional<ApplicationVersion> outstanding = app.lastSubmission(); + assertTrue(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); + assertEquals(Change.of(version).with(revision.get()), app.instance().change()); - assertTrue(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); - assertEquals("1.0.2-cafed00d", app1.deploymentStatus().outstandingChange(app1.instance().name()).application().get().id()); - app1.assertRunning(JobType.systemTest); - app1.assertRunning(JobType.stagingTest); - assertEquals(2, tester.jobs().active().size()); + tester.outstandingChangeDeployer().run(); + assertTrue(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); + assertEquals(Change.of(version).with(revision.get()), app.instance().change()); - deployer.maintain(); - tester.triggerJobs(); - assertEquals("No effect as job is in progress", 2, tester.jobs().active().size()); - assertEquals("1.0.2-cafed00d", app1.deploymentStatus().outstandingChange(app1.instance().name()).application().get().id()); - - app1.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1) - .runJob(JobType.stagingTest).runJob(JobType.systemTest); - assertEquals("Upgrade done", 0, tester.jobs().active().size()); - - deployer.maintain(); - tester.triggerJobs(); - assertEquals("1.0.2-cafed00d", app1.instance().change().application().get().id()); - List<Run> runs = tester.jobs().active(); - assertEquals(1, runs.size()); - app1.assertRunning(JobType.productionUsWest1); - assertFalse(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); + app.deploy(); + tester.outstandingChangeDeployer().run(); + assertFalse(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); + assertEquals(Change.of(outstanding.get()), app.instance().change()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index f9a976fcadc..0e61ffd3ea6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -167,7 +167,6 @@ public class UpgraderTest { // --- Failing application is repaired by changing the application, causing confidence to move above 'high' threshold // Deploy application change default0.submit(applicationPackage("default")); - default0.triggerJobs().jobAborted(stagingTest); default0.deploy(); tester.controllerTester().computeVersionStatus(); @@ -570,12 +569,12 @@ public class UpgraderTest { // A new revision is submitted and starts rolling out. app.submit(applicationPackage); - // production-us-central-1 isn't triggered, as the revision + platform is the new change to roll out. + // production-us-central-1 is re-triggered with upgrade until revision catches up. tester.triggerJobs(); - assertEquals(2, tester.jobs().active().size()); + assertEquals(3, tester.jobs().active().size()); app.runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1); // us-central-1 has an older version, and needs a new staging test to begin. - app.runJob(stagingTest); + app.runJob(stagingTest).triggerJobs().jobAborted(productionUsCentral1); // Retry will include revision. tester.triggerJobs(); // Triggers us-central-1 before platform upgrade is cancelled. // A new version is also released, cancelling the upgrade, since it is failing on a now outdated version. @@ -799,8 +798,10 @@ public class UpgraderTest { app.instance().change().application().get().id().equals(applicationVersion)); // Deployment completes - app.runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1).runJob(productionUsEast3); - assertTrue("All jobs consumed", tester.jobs().active().isEmpty()); + app.runJob(systemTest).runJob(stagingTest) + .runJob(productionUsWest1) + .runJob(productionUsEast3); + assertEquals("All jobs consumed", List.of(), tester.jobs().active()); for (Deployment deployment : app.instance().deployments().values()) { assertEquals(version, deployment.version()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index e7e54828a5d..320cf978e7f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -159,9 +159,9 @@ public class ApplicationSerializerTest { assertEquals(original.latestVersion().get().sourceUrl(), serialized.latestVersion().get().sourceUrl()); assertEquals(original.latestVersion().get().commit(), serialized.latestVersion().get().commit()); assertEquals(original.latestVersion().get().bundleHash(), serialized.latestVersion().get().bundleHash()); - // Check deployed versions are added + // Check indirectly deployed versions are added, directly deployed are removed assertEquals(original.versions(), serialized.versions()); - assertEquals(serialized.versions(), new TreeSet<>(Set.of(applicationVersion1, applicationVersion2))); + assertEquals(serialized.versions(), new TreeSet<>(Set.of(applicationVersion2))); assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm()); diff --git a/docproc/src/main/java/com/yahoo/docproc/DocprocService.java b/docproc/src/main/java/com/yahoo/docproc/DocprocService.java index 32f28ba1294..8cb78bd718f 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocprocService.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocprocService.java @@ -30,10 +30,10 @@ import java.util.logging.Logger; * * @author bratseth */ -//TODO Vespa 8 This class and a lot of other in this package should not be part of PublicAPI +//TODO: Vespa 8: This class and a lot of other in this package should not be part of PublicAPI public class DocprocService extends AbstractComponent { - private static Logger log = Logger.getLogger(DocprocService.class.getName()); + private static final Logger log = Logger.getLogger(DocprocService.class.getName()); private volatile DocprocExecutor executor; /** The processings currently in progress at this service */ diff --git a/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java index 934e1a281e5..877ecf52789 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java @@ -48,10 +48,10 @@ public abstract class DocumentProcessor extends ChainedComponent { private Map<Pair<String,String>, String> fieldMap = new HashMap<>(); /** For a doc type, the actual field name mapping to do */ - // TODO how to flush this when reconfig of schemamapping? Must solve - private Map<String, Map<String, String>> docMapCache = new HashMap<>(); + // TODO: How to flush this when reconfig of schemamapping? + private final Map<String, Map<String, String>> docMapCache = new HashMap<>(); - final boolean hasAnnotations; + private final boolean hasAnnotations; public DocumentProcessor() { hasAnnotations = getClass().getAnnotation(Accesses.class) != null; @@ -70,6 +70,34 @@ public abstract class DocumentProcessor extends ChainedComponent { */ public abstract Progress process(Processing processing); + /** Sets the schema map for field names */ + public void setFieldMap(Map<Pair<String, String>, String> fieldMap) { + this.fieldMap = fieldMap; + + } + + /** Schema map for field names (doctype,from)→to */ + public Map<Pair<String, String>, String> getFieldMap() { + return fieldMap; + } + + public Map<String, String> getDocMap(String docType) { + Map<String, String> cached = docMapCache.get(docType); + if (cached!=null) { + return cached; + } + Map<String, String> ret = new HashMap<>(); + for (Entry<Pair<String, String>, String> e : fieldMap.entrySet()) { + // Remember to include tuple if doctype is unset in mapping + if (docType.equals(e.getKey().getFirst()) || e.getKey().getFirst()==null || "".equals(e.getKey().getFirst())) { + ret.put(e.getKey().getSecond(), e.getValue()); + } + } + docMapCache.put(docType, ret); + return ret; + } + + @Override public String toString() { return "processor " + getId().stringValue(); } @@ -99,7 +127,7 @@ public abstract class DocumentProcessor extends ChainedComponent { */ public static final Progress PERMANENT_FAILURE = new Progress("permanent_failure"); - private String name; + private final String name; private Optional<String> reason = Optional.empty(); @@ -120,6 +148,7 @@ public abstract class DocumentProcessor extends ChainedComponent { return new Progress(this.name, reason); } + @Override public String toString() { return name; } @@ -128,16 +157,20 @@ public abstract class DocumentProcessor extends ChainedComponent { return reason; } + @Override public boolean equals(Object object) { return object instanceof Progress && ((Progress) object).name.equals(this.name); } + @Override public int hashCode() { return name.hashCode(); } + } public static final class LaterProgress extends Progress { + private final long delay; public static final long DEFAULT_LATER_DELAY = 20; //ms @@ -153,33 +186,7 @@ public abstract class DocumentProcessor extends ChainedComponent { public long getDelay() { return delay; } - } - - /** Sets the schema map for field names */ - public void setFieldMap(Map<Pair<String, String>, String> fieldMap) { - this.fieldMap = fieldMap; - - } - /** Schema map for field names (doctype,from)→to */ - public Map<Pair<String, String>, String> getFieldMap() { - return fieldMap; - } - - public Map<String, String> getDocMap(String docType) { - Map<String, String> cached = docMapCache.get(docType); - if (cached!=null) { - return cached; - } - Map<String, String> ret = new HashMap<>(); - for (Entry<Pair<String, String>, String> e : fieldMap.entrySet()) { - // Remember to include tuple if doctype is unset in mapping - if (docType.equals(e.getKey().getFirst()) || e.getKey().getFirst()==null || "".equals(e.getKey().getFirst())) { - ret.put(e.getKey().getSecond(), e.getValue()); - } - } - docMapCache.put(docType, ret); - return ret; } } diff --git a/docproc/src/main/java/com/yahoo/docproc/Processing.java b/docproc/src/main/java/com/yahoo/docproc/Processing.java index d2583fc5a6d..616ad3c9241 100644 --- a/docproc/src/main/java/com/yahoo/docproc/Processing.java +++ b/docproc/src/main/java/com/yahoo/docproc/Processing.java @@ -20,23 +20,23 @@ import java.util.Map; */ public class Processing { - /** The name of the service which owns this processing. Null is the same as "default" */ + /** The name of the service which owns this processing. Null is the same as "default". */ private String service = null; - /** The processors to call the next work is done on this processing */ + /** The processors to call the next work is done on this processing. */ private CallStack callStack = null; - /** The collection of documents or document updates processed by this. This is never null */ + /** The collection of documents or document updates processed by this. This is never null. */ private final List<DocumentOperation> documentOperations; /** * Documents or document updates which should be added to <code>documents</code> before * the next access, or null if documents or document updates have never been added to - * this processing + * this processing. */ private List<DocumentOperation> documentsToAdd = null; - /** The processing context variables */ + /** The processing context variables. */ private Map<String, Object> context = null; /** The endpoint of this processing. */ @@ -113,6 +113,7 @@ public class Processing { } /** + * Creates a Processing from a list of operations. * * @param service the unique name of the service processing this * @param documentsAndUpdates the document operation list. This <b>transfers ownership</b> of this list @@ -121,7 +122,9 @@ public class Processing { * This <b>transfers ownership</b> of this structure * to this class. The caller <i>must not</i> modify it */ - public static Processing createProcessingFromDocumentOperations(String service, List<DocumentOperation> documentsAndUpdates, CallStack callStack) { + public static Processing createProcessingFromDocumentOperations(String service, + List<DocumentOperation> documentsAndUpdates, + CallStack callStack) { return new Processing(service, documentsAndUpdates, callStack, null, false); } @@ -245,6 +248,15 @@ public class Processing { this.callStack = callStack; } + List<DocumentOperation> getOnceOperationsToBeProcessed() { + if (operationsGotten) + return Collections.emptyList(); + + operationsGotten = true; + return getDocumentOperations(); + } + + @Override public String toString() { String previousCall = ""; if (callStack != null) { @@ -266,11 +278,4 @@ public class Processing { } } - List<DocumentOperation> getOnceOperationsToBeProcessed() { - if (operationsGotten) - return Collections.emptyList(); - - operationsGotten = true; - return getDocumentOperations(); - } } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index acbedaf1156..2affcf12809 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -161,7 +161,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { String serviceName = requestContext.getServiceName(); DocprocService service = docprocServiceRegistry.getComponent(serviceName); - //No need to enqueue a task if the docproc chain is empty, just forward requestContext + // No need to enqueue a task if the docproc chain is empty, just forward requestContext if (service == null) { log.log(Level.SEVERE, "DocprocService for session '" + serviceName + "' not found, returning request '" + requestContext + "'."); diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index 146cd9cb988..655703d10ee 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -32,7 +32,7 @@ public class DocumentProcessingTask implements Runnable { private final List<Processing> processingsDone = new ArrayList<>(); private final DocumentProcessingHandler docprocHandler; - private RequestContext requestContext; + private final RequestContext requestContext; private final DocprocService service; private final ThreadPoolExecutor executor; diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java index 910d4b7961b..a17be4de9a5 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java @@ -107,4 +107,5 @@ class ProcessingFactory { processing.setVariable("timeout", message.getTimeRemaining()); return processing; } + } diff --git a/document/pom.xml b/document/pom.xml index 3faada08553..e2b9335ddc2 100644 --- a/document/pom.xml +++ b/document/pom.xml @@ -83,12 +83,10 @@ <finalName>${project.artifactId}-jar-with-dependencies</finalName> <filters> <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java index b76dee5efa9..1d2d8cfd309 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java @@ -81,6 +81,8 @@ public class VisitorIterator { protected static class DistributionRangeBucketSource implements BucketSource { private boolean flushActive = false; private int distributionBitCount; + private long totalBucketsSplit; + private long totalBucketsMerged; private final int slices; private final int sliceId; // Wouldn't need this if this were a non-static class, but do it for @@ -99,7 +101,9 @@ public class VisitorIterator { this.slices = slices; this.sliceId = sliceId; - progressToken = progress; + this.totalBucketsSplit = 0; + this.totalBucketsMerged = 0; + this.progressToken = progress; // New progress token (could also be empty, in which this is a // no-op anyway) @@ -281,6 +285,8 @@ public class VisitorIterator { bucketsMerged + " merge ops. Pending: " + pendingBefore + " -> " + p.getPendingBucketCount()); } + totalBucketsSplit += bucketsSplit; + totalBucketsMerged += bucketsMerged; } private void correctTruncatedBucketCursor() { diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 124703c24b8..aa256d0a291 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -104,12 +104,10 @@ <finalName>${project.artifactId}-jar-with-dependencies</finalName> <filters> <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 61f0bf6bbf8..150bfbdc137 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -47,35 +47,35 @@ public class Flags { public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Default limit for when to apply termwise query evaluation", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( "feed-sequencer-type", "THROUGHPUT", - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Selects type of sequenced executor used for feeding in proton, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment (requires restart)", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag FEED_TASK_LIMIT = defineIntFlag( "feed-task-limit", 1000, - List.of("geirst, baldersheim"), "2021-10-14", "2022-02-01", + List.of("geirst, baldersheim"), "2021-10-14", "2022-06-01", "The task limit used by the executors handling feed in proton", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag FEED_MASTER_TASK_LIMIT = defineIntFlag( "feed-master-task-limit", 1000, - List.of("geirst, baldersheim"), "2021-11-18", "2022-02-01", + List.of("geirst, baldersheim"), "2021-11-18", "2022-06-01", "The task limit used by the master thread in each document db in proton. Ignored when set to 0.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag SHARED_FIELD_WRITER_EXECUTOR = defineStringFlag( "shared-field-writer-executor", "NONE", - List.of("geirst, baldersheim"), "2021-11-05", "2022-02-01", + List.of("geirst, baldersheim"), "2021-11-05", "2022-06-01", "Whether to use a shared field writer executor for the document database(s) in proton. " + "Valid values: NONE, INDEX, INDEX_AND_ATTRIBUTE, DOCUMENT_DB", "Takes effect at redeployment (requires restart)", @@ -83,42 +83,42 @@ public class Flags { public static final UnboundIntFlag MAX_UNCOMMITTED_MEMORY = defineIntFlag( "max-uncommitted-memory", 130000, - List.of("geirst, baldersheim"), "2021-10-21", "2022-02-01", + List.of("geirst, baldersheim"), "2021-10-21", "2022-06-01", "Max amount of memory holding updates to an attribute before we do a commit.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag RESPONSE_SEQUENCER_TYPE = defineStringFlag( "response-sequencer-type", "ADAPTIVE", - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Selects type of sequenced executor used for mbus responses, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag RESPONSE_NUM_THREADS = defineIntFlag( "response-num-threads", 2, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Number of threads used for mbus responses, default is 2, negative number = numcores/4", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_COMMUNICATIONMANAGER_THREAD = defineFeatureFlag( "skip-communicationmanager-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Should we skip the communicationmanager thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_MBUS_REQUEST_THREAD = defineFeatureFlag( "skip-mbus-request-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Should we skip the mbus request thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_MBUS_REPLY_THREAD = defineFeatureFlag( "skip-mbus-reply-thread", false, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Should we skip the mbus reply thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -132,35 +132,35 @@ public class Flags { public static final UnboundBooleanFlag USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE = defineFeatureFlag( "async-message-handling-on-schedule", false, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "Optionally deliver async messages in own thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag FEED_CONCURRENCY = defineDoubleFlag( "feed-concurrency", 0.5, - List.of("baldersheim"), "2020-12-02", "2022-02-01", + List.of("baldersheim"), "2020-12-02", "2022-06-01", "How much concurrency should be allowed for feed", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( "enable-feed-block-in-distributor", true, - List.of("geirst"), "2021-01-27", "2022-01-31", + List.of("geirst"), "2021-01-27", "2022-06-01", "Enables blocking of feed in the distributor if resource usage is above limit on at least one content node", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT = defineFeatureFlag( "container-dump-heap-on-shutdown-timeout", false, - List.of("baldersheim"), "2021-09-25", "2022-02-01", + List.of("baldersheim"), "2021-09-25", "2022-06-01", "Will trigger a heap dump during if container shutdown times out", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag CONTAINER_SHUTDOWN_TIMEOUT = defineDoubleFlag( "container-shutdown-timeout", 50.0, - List.of("baldersheim"), "2021-09-25", "2022-02-01", + List.of("baldersheim"), "2021-09-25", "2022-06-01", "Timeout for shutdown of a jdisc container", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -209,7 +209,7 @@ public class Flags { public static final UnboundIntFlag METRICSPROXY_NUM_THREADS = defineIntFlag( "metricsproxy-num-threads", 2, - List.of("balder"), "2021-09-01", "2022-02-01", + List.of("balder"), "2021-09-01", "2022-06-01", "Number of threads for metrics proxy", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -288,7 +288,7 @@ public class Flags { public static final UnboundBooleanFlag IGNORE_THREAD_STACK_SIZES = defineFeatureFlag( "ignore-thread-stack-sizes", false, - List.of("arnej"), "2021-11-12", "2022-01-31", + List.of("arnej"), "2021-11-12", "2022-06-01", "Whether C++ thread creation should ignore any requested stack size", "Triggers restart, takes effect immediately", ZONE_ID, APPLICATION_ID); diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml index 86dc9f0fbb8..535de8858ea 100644 --- a/indexinglanguage/pom.xml +++ b/indexinglanguage/pom.xml @@ -67,12 +67,10 @@ <finalName>${project.artifactId}-jar-with-dependencies</finalName> <filters> <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> diff --git a/logserver/pom.xml b/logserver/pom.xml index 19f422f3a86..ca1a891633d 100644 --- a/logserver/pom.xml +++ b/logserver/pom.xml @@ -65,12 +65,10 @@ <finalName>${project.artifactId}-jar-with-dependencies</finalName> <filters> <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> diff --git a/messagebus/pom.xml b/messagebus/pom.xml index ff55a4eca96..124368cb136 100644 --- a/messagebus/pom.xml +++ b/messagebus/pom.xml @@ -69,12 +69,10 @@ <finalName>${project.artifactId}-jar-with-dependencies</finalName> <filters> <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> diff --git a/security-tools/pom.xml b/security-tools/pom.xml index 7f5d22b4b2b..ad5aa73d546 100644 --- a/security-tools/pom.xml +++ b/security-tools/pom.xml @@ -51,13 +51,10 @@ <finalName>${project.artifactId}-jar-with-dependencies</finalName> <filters> <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - <exclude>META-INF/versions/*/module-info.class</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java index 2fc7e5af7b4..ac859359bfa 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliArguments.java @@ -54,6 +54,7 @@ class CliArguments { private static final String SILENT_OPTION = "silent"; private static final String VERSION_OPTION = "version"; private static final String STDIN_OPTION = "stdin"; + private static final String DOOM_OPTION = "max-failure-seconds"; private final CommandLine arguments; @@ -149,6 +150,8 @@ class CliArguments { OptionalInt traceLevel() throws CliArgumentsException { return intValue(TRACE_OPTION); } + OptionalInt doomSeconds() throws CliArgumentsException { return intValue(DOOM_OPTION); } + Optional<Duration> timeout() throws CliArgumentsException { OptionalDouble timeout = doubleValue(TIMEOUT_OPTION); return timeout.isPresent() @@ -259,7 +262,7 @@ class CliArguments { .build()) .addOption(Option.builder() .longOpt(BENCHMARK_OPTION) - .desc("Enable benchmark mode") + .desc("Print statistics to stdout when done") .build()) .addOption(Option.builder() .longOpt(ROUTE_OPTION) @@ -279,8 +282,14 @@ class CliArguments { .type(Number.class) .build()) .addOption(Option.builder() - .longOpt(STDIN_OPTION) - .desc("Read JSON input from standard input") + .longOpt(STDIN_OPTION) + .desc("Read JSON input from standard input") + .build()) + .addOption(Option.builder() + .longOpt(DOOM_OPTION) + .desc("Exit if specified number of seconds ever pass without any successful operations. Disabled by default") + .hasArg() + .type(Number.class) .build()) .addOption(Option.builder() .longOpt(DRYRUN_OPTION) @@ -292,7 +301,7 @@ class CliArguments { .build()) .addOption(Option.builder() .longOpt(SILENT_OPTION) - .desc("Disable periodic status printing") + .desc("Disable periodic status printing to stderr") .build()) .addOption(Option.builder() .longOpt(SHOW_ERRORS_OPTION) diff --git a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java index 7e036b8dec3..5e904b37588 100644 --- a/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java +++ b/vespa-feed-client-cli/src/main/java/ai/vespa/feed/client/impl/CliClient.java @@ -19,6 +19,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; import java.nio.file.Files; +import java.time.Duration; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -94,9 +95,7 @@ public class CliClient { }); latch.await(); - if (cliArgs.benchmarkModeEnabled()) { - printBenchmarkResult(System.nanoTime() - startNanos, successes.get(), failures.get(), feedClient.stats(), systemOut); - } + printBenchmarkResult(System.nanoTime() - startNanos, successes.get(), failures.get(), feedClient.stats(), cliArgs.benchmarkModeEnabled() ? systemOut : systemError); if (fatal.get() != null) throw fatal.get(); } return 0; @@ -137,6 +136,8 @@ public class CliClient { cliArgs.caCertificates().ifPresent(builder::setCaCertificatesFile); cliArgs.headers().forEach(builder::addRequestHeader); builder.setDryrun(cliArgs.dryrunEnabled()); + cliArgs.doomSeconds().ifPresent(doom -> builder.setCircuitBreaker(new GracePeriodCircuitBreaker(Duration.ofSeconds(10), + Duration.ofSeconds(doom)))); return builder.build(); } diff --git a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java index 19b93c3172b..fe3dc465814 100644 --- a/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java +++ b/vespa-feed-client-cli/src/test/java/ai/vespa/feed/client/impl/CliArgumentsTest.java @@ -25,7 +25,8 @@ class CliArgumentsTest { "--max-streams-per-connection=128", "--certificate=cert.pem", "--private-key=key.pem", "--ca-certificates=ca-certs.pem", "--disable-ssl-hostname-verification", "--header=\"My-Header: my-value\"", "--header", "Another-Header: another-value", "--benchmark", - "--route=myroute", "--timeout=0.125", "--trace=9", "--verbose", "--silent", "--show-errors", "--show-all"}); + "--route=myroute", "--timeout=0.125", "--trace=9", "--verbose", "--silent", + "--show-errors", "--show-all", "--max-failure-seconds=30"}); assertEquals(URI.create("https://vespa.ai:4443/"), args.endpoint()); assertEquals(Paths.get("feed.json"), args.inputFile().get()); assertEquals(10, args.connections().getAsInt()); @@ -43,6 +44,7 @@ class CliArgumentsTest { assertEquals("myroute", args.route().get()); assertEquals(Duration.ofMillis(125), args.timeout().get()); assertEquals(9, args.traceLevel().getAsInt()); + assertEquals(30, args.doomSeconds().getAsInt()); assertTrue(args.verboseSpecified()); assertTrue(args.showErrors()); assertTrue(args.showSuccesses()); diff --git a/vespa-feed-client-cli/src/test/resources/help.txt b/vespa-feed-client-cli/src/test/resources/help.txt index 67b83c07699..323206ab128 100644 --- a/vespa-feed-client-cli/src/test/resources/help.txt +++ b/vespa-feed-client-cli/src/test/resources/help.txt @@ -1,6 +1,7 @@ usage: vespa-feed-client <options> Vespa feed client - --benchmark Enable benchmark mode + --benchmark Print statistics to stdout when + done --ca-certificates <arg> Path to file containing CA X.509 certificates encoded as PEM --certificate <arg> Path to PEM encoded X.509 @@ -16,6 +17,10 @@ Vespa feed client --header <arg> HTTP header on the form 'Name: value' --help + --max-failure-seconds <arg> Exit if specified number of + seconds ever pass without any + successful operations. Disabled + by default --max-streams-per-connection <arg> Maximum number of concurrent streams per HTTP/2 connection --private-key <arg> Path to PEM/PKCS#8 encoded @@ -27,6 +32,7 @@ Vespa feed client --show-errors Print every feed operation failure --silent Disable periodic status printing + to stderr --stdin Read JSON input from standard input --timeout <arg> Feed operation timeout (in diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 3cf669f924a..42c5525c321 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -191,9 +191,8 @@ <filter> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> diff --git a/vespa-http-client/pom.xml b/vespa-http-client/pom.xml index b25f54362ed..f2a21a259b0 100644 --- a/vespa-http-client/pom.xml +++ b/vespa-http-client/pom.xml @@ -172,12 +172,10 @@ <outputFile>target/${project.artifactId}-jar-with-dependencies.jar</outputFile> <filters> <filter> - <!-- Don't include signature files in uber jar (most likely from bouncycastle). --> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> diff --git a/vespa_feed_perf/pom.xml b/vespa_feed_perf/pom.xml index 6c25b4f6329..25b57cdcbc9 100644 --- a/vespa_feed_perf/pom.xml +++ b/vespa_feed_perf/pom.xml @@ -64,12 +64,10 @@ <finalName>${project.artifactId}-jar-with-dependencies</finalName> <filters> <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml index d8530fd9d82..57f0a0b6125 100644 --- a/vespaclient-java/pom.xml +++ b/vespaclient-java/pom.xml @@ -77,12 +77,10 @@ <finalName>${project.artifactId}-jar-with-dependencies</finalName> <filters> <filter> - <!-- Don't include signature files in uber jar (most likely from bouncycastle). --> <artifact>*:*</artifact> <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/**</exclude> + <exclude>module-info.class</exclude> </excludes> </filter> </filters> |