blob: e25db10a8cd099ae775f709c099d216cac889d63 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.deployment;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.vespa.curator.Lock;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob;
import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Deque;
import java.util.List;
import java.util.Optional;
import java.util.logging.Logger;
/**
* Stores a queue for each type of job, and offers jobs from each of these to a periodic
* polling mechanism which is responsible for triggering the offered jobs in an external build service.
*
* @author jvenstad
* @author mpolden
*/
public class PolledBuildSystem implements BuildSystem {
private static final Logger log = Logger.getLogger(PolledBuildSystem.class.getName());
// The number of jobs to offer, on each poll, for zones that have limited capacity
private static final int maxCapacityConstrainedJobsToOffer = 2;
private final Controller controller;
private final CuratorDb curator;
public PolledBuildSystem(Controller controller, CuratorDb curator) {
this.controller = controller;
this.curator = curator;
}
@Override
public void addJob(ApplicationId application, JobType jobType, boolean first) {
try (Lock lock = curator.lockJobQueues()) {
Deque<ApplicationId> queue = curator.readJobQueue(jobType);
if ( ! queue.contains(application)) {
if (first) {
queue.addFirst(application);
} else {
queue.add(application);
}
}
curator.writeJobQueue(jobType, queue);
}
}
@Override
public List<BuildJob> jobs() {
return getJobs(false);
}
@Override
public List<BuildJob> takeJobsToRun() {
return getJobs(true);
}
@Override
public void removeJobs(ApplicationId application) {
try (Lock lock = curator.lockJobQueues()) {
for (JobType jobType : JobType.values()) {
Deque<ApplicationId> queue = curator.readJobQueue(jobType);
while (queue.remove(application)) {
// keep removing until not found
}
curator.writeJobQueue(jobType, queue);
}
}
}
private List<BuildJob> getJobs(boolean removeFromQueue) {
int capacityConstrainedJobsOffered = 0;
try (Lock lock = curator.lockJobQueues()) {
List<BuildJob> jobsToRun = new ArrayList<>();
for (JobType jobType : JobType.values()) {
Deque<ApplicationId> queue = curator.readJobQueue(jobType);
for (ApplicationId a : queue) {
ApplicationId application = removeFromQueue ? queue.poll() : a;
Optional<Long> projectId = projectId(application);
if (projectId.isPresent()) {
jobsToRun.add(new BuildJob(projectId.get(), jobType.jobName()));
} else {
log.warning("Not queuing " + jobType.jobName() + " for " + application.toShortString() +
" because project ID is missing");
}
// Return a limited number of jobs at a time for capacity constrained zones
if (removeFromQueue && isCapacityConstrained(jobType) &&
++capacityConstrainedJobsOffered >= maxCapacityConstrainedJobsToOffer) {
break;
}
}
if (removeFromQueue)
curator.writeJobQueue(jobType, queue);
}
return Collections.unmodifiableList(jobsToRun);
}
}
private Optional<Long> projectId(ApplicationId applicationId) {
return controller.applications().require(applicationId).deploymentJobs().projectId();
}
private static boolean isCapacityConstrained(JobType jobType) {
return jobType == JobType.stagingTest || jobType == JobType.systemTest;
}
}
|
