diff options
6 files changed, 168 insertions, 7 deletions
diff --git a/container-core/src/main/java/com/yahoo/restapi/Path.java b/container-core/src/main/java/com/yahoo/restapi/Path.java index 65f0bab9a8d..7f78572f2d7 100644 --- a/container-core/src/main/java/com/yahoo/restapi/Path.java +++ b/container-core/src/main/java/com/yahoo/restapi/Path.java @@ -19,7 +19,7 @@ import java.util.stream.Collectors; * If the path spec ends with /{*}, it will match urls with any rest path. * The rest path (not including the trailing slash) will be available as getRest(). * - * Note that for convenience in common use this has state which is changes as a side effect of each matches + * Note that for convenience in common use this has state which changes as a side effect of each matches * invocation. It is therefore for single thread use. * * @author bratseth @@ -40,6 +40,8 @@ public class Path { } /** + * Parses the path according to pathSpec - must be called prior to {@link #get} + * * Returns whether this path matches the given template string. * If the given template has placeholders, their values (accessible by get) are reset by calling this, * whether or not the path matches the given template. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 9ced75303ff..b35b166f6b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -169,14 +169,14 @@ public class JobController { } /** Registers the given application, such that it may have deployment jobs run here. */ - void register(ApplicationId id) { + public void register(ApplicationId id) { controller.applications().lockIfPresent(id, application -> controller.applications().store(application.withBuiltInternally(true))); } /** Accepts and stores a new application package and test jar pair under a generated application version key. */ public ApplicationVersion submit(ApplicationId id, SourceRevision revision, - byte[] applicationPackage, byte[] applicationTestJar) { + byte[] applicationPackage, byte[] applicationTestPackage) { AtomicReference<ApplicationVersion> version = new AtomicReference<>(); controller.applications().lockOrThrow(id, application -> { controller.applications().store(application.withBuiltInternally(true)); @@ -185,6 +185,9 @@ public class JobController { version.set(ApplicationVersion.from(revision, run)); // TODO smorgrav: Store the pair. +// controller.applications().artifacts().putApplicationPackage(id, version.toString(), applicationPackage); +// controller.applications().artifacts().putTesterPackage( +// InternalStepRunner.testerOf(id), version.toString(), applicationTestPackage); notifyOfNewSubmission(id, revision, run); }); @@ -265,7 +268,7 @@ public class JobController { private void notifyOfNewSubmission(ApplicationId id, SourceRevision revision, long number) { DeploymentJobs.JobReport report = new DeploymentJobs.JobReport(id, JobType.component, - Long.MAX_VALUE, // TODO jvenstad: Clean up this! + 1, number, Optional.of(revision), Optional.empty()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index a91cc905add..1fd32524c88 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -72,15 +72,23 @@ public class RunStatus { return id; } - /** Returns an unmodifiable view of the status of all steps in this run. */ + /** Returns an unmodifiable view of the status of all steps in this run. + * TODO maybe reflect in the signature that the map is a EnumMap or at least behaves as a sorted map? + * */ public Map<Step, Step.Status> steps() { return steps; } /** Returns the final result of this run, if it has ended. */ public Optional<RunResult> result() { - // TODO jvenstad: To implement, or not ... If so, base on status. - throw new AssertionError(); + + // No result of not finished yet + if (!hasEnded()) return Optional.empty(); + + // If any steps has failed - then we need to figure out what - for now return fixed error result + if (hasFailed()) return Optional.of(RunResult.testError); + + return Optional.of(RunResult.success); } /** Returns the instant at which this run began. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 8fc4abefdc0..e1e2281c5ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -66,6 +66,9 @@ public enum Step { public List<Step> prerequisites() { return prerequisites; } + public static Step last() { + return report; + } public enum Status { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index b8f2ccad879..622761bc64f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -49,6 +49,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; @@ -63,6 +64,8 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.restapi.Path; @@ -88,6 +91,7 @@ import java.security.Principal; import java.time.DayOfWeek; import java.time.Duration; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -168,6 +172,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) + return JobControllerApiHandlerHelper.jobTypeResponse(jobTypes(path), latestRuns(path), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) + return JobControllerApiHandlerHelper.runStatusResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) + return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); @@ -1204,4 +1214,34 @@ public class ApplicationApiHandler extends LoggingRequestHandler { message + ": No NToken provided")); } + private ApplicationId appIdFromPath(Path path) { + return ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance")); + } + + private JobType jobTypeFromPath(Path path) { + return JobType.fromJobName(path.get("jobtype")); + } + + private RunId runIdFromPath(Path path) { + long number = Long.parseLong(path.get("number")); + return new RunId(appIdFromPath(path), jobTypeFromPath(path), number); + } + + private List<JobType> jobTypes(Path path) { + ApplicationId appId = appIdFromPath(path); + DeploymentSpec deploymentSpec = controller.applications().get(appId).get().deploymentSpec(); + DeploymentSteps deploymentSteps = new DeploymentSteps(deploymentSpec, controller::system); + return deploymentSteps.jobs(); + } + + private Map<JobType, RunStatus> latestRuns(Path path) { + Map<JobType, RunStatus> jobMap = new HashMap<>(); + ApplicationId appId = appIdFromPath(path); + controller.jobController().jobs(appId) + .forEach(jobType -> jobMap.put(jobType, controller.jobController() + .last(appId, jobType) + .orElseThrow(() -> new RuntimeException("This is a data violation right?")))); + + return jobMap; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java new file mode 100644 index 00000000000..bff67501ea3 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -0,0 +1,105 @@ +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.NotExistsException; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.deployment.JobController; +import com.yahoo.vespa.hosted.controller.deployment.RunDetails; +import com.yahoo.vespa.hosted.controller.deployment.RunStatus; +import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; + +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Implements the REST API for the job controller delegated from the Application API. + * + * @see JobController + * @see ApplicationApiHandler + */ +class JobControllerApiHandlerHelper { + + /** + * @return Response with all job types that have recorded runs for the application _and_ the status for the last run of that type + */ + static HttpResponse jobTypeResponse(List<JobType> sortedJobs, Map<JobType, RunStatus> lastStatus, URI baseUriForJobs) { + Slime slime = new Slime(); + Cursor responseObject = slime.setObject(); + Cursor jobArray = responseObject.setArray("jobs"); + + sortedJobs.forEach(jobType -> + jobTypeToSlime(jobArray.addObject(), jobType, Optional.ofNullable(lastStatus.get(jobType)), baseUriForJobs)); + return new SlimeJsonResponse(slime); + } + + private static void jobTypeToSlime(Cursor cursor, JobType jobType, Optional<RunStatus> runStatus, URI baseUriForJobs) { + Cursor jobObject = cursor.setObject(jobType.jobName()); + + // Url that are specific to the jobtype + String jobTypePath = baseUriForJobs.getPath() + "/" + jobType.jobName(); + URI baseUriForJobType = baseUriForJobs.resolve(jobTypePath); + jobObject.setString("url", baseUriForJobType.toString()); + + // Add the last run status for the jobtype if present + runStatus.ifPresent(status -> { + Cursor lastObject = jobObject.setObject("last"); + runStatusToSlime(lastObject, status, baseUriForJobType); + }); + } + + /** + * @return Response with the runstatuses for a specific jobtype + */ + static HttpResponse runStatusResponse(Map<RunId, RunStatus> runStatuses, URI baseUriForJobType) { + Slime slime = new Slime(); + Cursor cursor = slime.setObject(); + + runStatuses.forEach((runid, runstatus) -> runStatusToSlime(cursor.setObject(Long.toString(runid.number())), runstatus, baseUriForJobType)); + + return new SlimeJsonResponse(slime); + } + + private static void runStatusToSlime(Cursor cursor, RunStatus runStatus, URI baseUriForJobType) { + runStatus.result().ifPresent(result -> cursor.setString("result", result.name())); + runStatus.end().ifPresent(instant -> cursor.setString("end", instant.toString())); + + Cursor stepsArray = cursor.setArray("steps"); + runStatus.steps().forEach((step, status) -> { + Cursor stepObject = stepsArray.addObject(); + stepObject.setString(step.name(), status.name()); + }); + + cursor.setString("start", runStatus.start().toString()); + cursor.setLong("id", runStatus.id().number()); + String logsPath = baseUriForJobType.getPath() + "/run/" + runStatus.id().number(); + cursor.setString("logs", baseUriForJobType.resolve(logsPath).toString()); + } + + /** + * @return Response with logs from a single run + */ + static HttpResponse runDetailsResponse(JobController jobController, RunId runId) { + Slime slime = new Slime(); + Cursor logsObject = slime.setObject(); + + RunDetails runDetails = jobController.details(runId).orElseThrow(() -> + new NotExistsException(String.format( + "No run details exist for application: %s, job type: %s, number: %d", + runId.application().toShortString(), runId.type().jobName(), runId.number()))); + for (Step step : Step.values()) { + runDetails.get(step).ifPresent(stepLog -> logsObject.setString(step.name(), stepLog)); + } + + return new SlimeJsonResponse(slime); + } +} + |