From ecd5f5d13936dd097b12766cb95a568ad5b073a9 Mon Sep 17 00:00:00 2001 From: Melodi <62381002+Melodi17@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:04:59 +1000 Subject: feat: lots of stuff --- marlin.sln.DotSettings.user | 1 + src/marlin.agents.shell/CommandRunner.cs | 38 ++ src/marlin.agents.shell/JobUnitOfWork.cs | 130 +++++ src/marlin.agents.shell/MarlinApiClient.cs | 33 +- src/marlin.agents.shell/Program.cs | 27 +- src/marlin.agents.shell/marlin.agents.shell.csproj | 2 + src/marlin.core/Class1.cs | 3 - src/marlin.core/contracts/InstanceService.cs | 43 ++ src/marlin.core/functional/GitBinaryClient.cs | 9 +- src/marlin.core/functional/IGitClient.cs | 1 + src/marlin.core/marlin.core.csproj | 1 + src/marlin.core/models/ISourceSecretStore.cs | 1 + src/marlin.core/models/JobStatus.cs | 10 + src/marlin.core/models/plan/ExecutionCommand.cs | 7 +- src/marlin.core/util/CommonPaths.cs | 10 + .../Controllers/Api/DispatchController.cs | 69 +++ .../Controllers/Api/InstanceGrpcService.cs | 54 ++ src/marlin.web/Controllers/DispatchController.cs | 71 --- src/marlin.web/Controllers/JobsController.cs | 4 + src/marlin.web/Data/ApplicationDbContext.cs | 2 + .../20261027095603_AddLogEvent.Designer.cs | 565 +++++++++++++++++++++ .../Migrations/20261027095603_AddLogEvent.cs | 67 +++ ...0261027130151_JobStatusUpdatesTable.Designer.cs | 565 +++++++++++++++++++++ .../20261027130151_JobStatusUpdatesTable.cs | 78 +++ .../ApplicationDbContextModelSnapshot.cs | 61 ++- src/marlin.web/Models/Job.cs | 13 +- src/marlin.web/Models/LogEventRecord.cs | 16 + src/marlin.web/Program.cs | 8 +- .../Services/VersionControlSyncService.cs | 2 +- src/marlin.web/Views/Jobs/Details.cshtml | 25 +- src/marlin.web/marlin.web.csproj | 2 + 31 files changed, 1821 insertions(+), 97 deletions(-) create mode 100644 src/marlin.agents.shell/CommandRunner.cs create mode 100644 src/marlin.agents.shell/JobUnitOfWork.cs delete mode 100644 src/marlin.core/Class1.cs create mode 100644 src/marlin.core/contracts/InstanceService.cs create mode 100644 src/marlin.core/models/JobStatus.cs create mode 100644 src/marlin.core/util/CommonPaths.cs create mode 100644 src/marlin.web/Controllers/Api/DispatchController.cs create mode 100644 src/marlin.web/Controllers/Api/InstanceGrpcService.cs delete mode 100644 src/marlin.web/Controllers/DispatchController.cs create mode 100644 src/marlin.web/Migrations/20261027095603_AddLogEvent.Designer.cs create mode 100644 src/marlin.web/Migrations/20261027095603_AddLogEvent.cs create mode 100644 src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.Designer.cs create mode 100644 src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.cs create mode 100644 src/marlin.web/Models/LogEventRecord.cs diff --git a/marlin.sln.DotSettings.user b/marlin.sln.DotSettings.user index 2651527..252ac25 100644 --- a/marlin.sln.DotSettings.user +++ b/marlin.sln.DotSettings.user @@ -1,4 +1,5 @@  ForceIncluded ForceIncluded + ForceIncluded ForceIncluded \ No newline at end of file diff --git a/src/marlin.agents.shell/CommandRunner.cs b/src/marlin.agents.shell/CommandRunner.cs new file mode 100644 index 0000000..d4fcbb3 --- /dev/null +++ b/src/marlin.agents.shell/CommandRunner.cs @@ -0,0 +1,38 @@ +using System.Diagnostics; + +public static class CommandRunner +{ + public static async Task RunAsync(string command, Action onStdout, Action onStderr) + { + var psi = new ProcessStartInfo + { + FileName = GetShell(), + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + var process = new Process { StartInfo = psi }; + process.OutputDataReceived += (_, e) => { if (e.Data != null) onStdout(e.Data); }; + process.ErrorDataReceived += (_, e) => { if (e.Data != null) onStderr(e.Data); }; + + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + // Write the command to stdin and close it + await process.StandardInput.WriteLineAsync(command); + await process.StandardInput.WriteLineAsync("exit $?"); // Exit with the command's exit code + process.StandardInput.Close(); + + await process.WaitForExitAsync(); + return process.ExitCode; + } + + private static string GetShell() + { + return OperatingSystem.IsWindows() ? "cmd.exe" : "/bin/bash"; + } +} \ No newline at end of file diff --git a/src/marlin.agents.shell/JobUnitOfWork.cs b/src/marlin.agents.shell/JobUnitOfWork.cs new file mode 100644 index 0000000..85df806 --- /dev/null +++ b/src/marlin.agents.shell/JobUnitOfWork.cs @@ -0,0 +1,130 @@ +namespace marlin.agents.shell; + +using System.Threading.Channels; +using core.contracts; +using core.functional; +using core.models; +using core.models.meta; +using core.models.plan; +using core.util; +using Grpc.Net.Client; +using ProtoBuf.Grpc.Client; + +public class JobUnitOfWork +{ + private static string _jobInstanceDir = Path.Combine(CommonPaths.AgentRoot, "instance"); + + private readonly MarlinApiClient _apiClient; + private readonly IGitClient _gitClient; + private readonly DispatchJob _job; + private readonly string _checkoutPath; + + private GrpcChannel _channel; + private IInstanceService _instanceService; + + private Channel _logQueue; + + public JobUnitOfWork(DispatchJob job, MarlinApiClient apiClient, IGitClient gitClient) + { + this._job = job; + this._apiClient = apiClient; + this._gitClient = gitClient; + + this._checkoutPath = Path.Combine(_jobInstanceDir, job.Id.ToString("N")); + } + + public async Task Run() + { + this._channel = this._apiClient.CreateJobScopedRpc(this._job); + this._instanceService = this._channel.CreateGrpcService(); + this._logQueue = Channel.CreateUnbounded(); + + var drainTask = Task.Run(this.DrainLogEvents); + + await StatusUpdate(JobStatus.Active); + + await this.CheckoutSource(); + var plan = await this.BuildPlan(); + var success = await this.ExecutePlan(plan); + + await StatusUpdate(success ? JobStatus.Success : JobStatus.Failed); + + this._logQueue.Writer.Complete(); + await drainTask; + } + + private async Task CheckoutSource() + { + Directory.CreateDirectory(_jobInstanceDir); + if (Path.Exists(this._checkoutPath)) + Directory.Delete(this._checkoutPath, true); + + await this._gitClient.Checkout(this._checkoutPath, this._job.Vcs, this._job.Ref, this._job.Commit); + } + + private async Task DrainLogEvents() + { + await foreach (var evt in this._logQueue.Reader.ReadAllAsync()) + await this._instanceService.SendLogEvent(evt); + } + + private async Task BuildPlan() + { + WorkflowBuilder builder = new(); + builder.LoadFromDirectory(Path.Join(this._checkoutPath, ".marlin")); + + Workflow workflow = builder.Build(); + Recipe recipe = workflow.Recipes[this._job.Recipe]; + ExecutionPlan plan = ExecutionPlan.Create(workflow, recipe); + + return plan; + } + + private async Task ExecutePlan(ExecutionPlan plan) + { + Dictionary results = new(); + + foreach (var component in plan.Components) + { + if (component.ResultSelector != null) + { + if (!results.ContainsKey(component.ResultSelector.Command.Id)) + // prerequisite step did not run + continue; + + var storedResult = results[component.ResultSelector.Command.Id]; + if (!storedResult.Matches(component.ResultSelector.Result)) + // prerequisite step returned incorrect error code + continue; + } + + Console.WriteLine(component); + Task run = CommandRunner.RunAsync( + component.Command, + o => Log(o, false, component), + o => Log(o, true, component)); + + var result = await run; + results[component.Id] = result == 0 ? CommandResult.Success : CommandResult.Failure; + } + + return results.All(x => x.Value == CommandResult.Success); + } + + private void Log(string message, bool error, ExecutionCommand component) + { + this._logQueue.Writer.TryWrite( + new LogEventRequest + { + At = DateTime.UtcNow, + Error = error, + Message = message, + StepId = component.Id + }); + } + + private async Task StatusUpdate(JobStatus status) + { + await this._instanceService.SendStatusUpdate(new StatusUpdateRequest { Status = status }); + } +} \ No newline at end of file diff --git a/src/marlin.agents.shell/MarlinApiClient.cs b/src/marlin.agents.shell/MarlinApiClient.cs index aa4f013..900101c 100644 --- a/src/marlin.agents.shell/MarlinApiClient.cs +++ b/src/marlin.agents.shell/MarlinApiClient.cs @@ -1,16 +1,28 @@ namespace marlin.agents.shell; using System.Net.Http.Json; +using core.contracts; using core.models; +using Grpc.Core; +using Grpc.Net.Client; +using ProtoBuf.Grpc.Client; public class MarlinApiClient { + private readonly string _baseUrl; + private readonly Guid _agentId; + private readonly string _agentToken; + private readonly HttpClient _http; public MarlinApiClient(string baseUrl, Guid agentId, string agentToken) { - _http = new HttpClient { BaseAddress = new Uri(baseUrl) }; - _http.DefaultRequestHeaders.Add("Agent", $"{agentId} {agentToken}"); + this._baseUrl = baseUrl; + this._agentId = agentId; + this._agentToken = agentToken; + + _http = new HttpClient { BaseAddress = new Uri(this._baseUrl) }; + _http.DefaultRequestHeaders.Add("Agent", $"{this._agentId} {this._agentToken}"); } public static MarlinApiClient FromEnvironment() @@ -35,4 +47,21 @@ public class MarlinApiClient public async Task QueueNextJob(CancellationToken ct = default) => await _http.GetFromJsonAsync("api/dispatch/queue", ct); + + public GrpcChannel CreateJobScopedRpc(DispatchJob job) + { + var credentials = CallCredentials.FromInterceptor( + async (_, metadata) => + { + metadata.Add("Agent", $"{this._agentId} {this._agentToken}"); + metadata.Add("Job", $"{job.Id} {job.Token}"); + }); + + var channel = GrpcChannel.ForAddress(this._baseUrl, new GrpcChannelOptions + { + Credentials = ChannelCredentials.Create(new SslCredentials(), credentials) + }); + + return channel; + } } diff --git a/src/marlin.agents.shell/Program.cs b/src/marlin.agents.shell/Program.cs index 02a0bc9..75ea8ad 100644 --- a/src/marlin.agents.shell/Program.cs +++ b/src/marlin.agents.shell/Program.cs @@ -1,9 +1,20 @@ using System.Runtime.InteropServices; +using System.Threading.Channels; +using Grpc.Net.Client; using marlin.agents.shell; +using marlin.core.contracts; +using marlin.core.functional; using marlin.core.models; +using marlin.core.models.meta; +using marlin.core.models.plan; +using marlin.core.util; +using ProtoBuf.Grpc.Client; DotNetEnv.Env.NoClobber().TraversePath().Load(); +var jobInstanceDir = Path.Combine(CommonPaths.AgentRoot, "instance"); +Directory.CreateDirectory(jobInstanceDir); + var client = MarlinApiClient.FromEnvironment(); var capabilities = new CapabilitySet(); @@ -16,5 +27,17 @@ AgentConfiguration config = await client.Handshake(status); Console.WriteLine($"IAM: {config.Name}"); Console.WriteLine($"MCJ: {config.MaxConcurrentJobs}"); -var job = await client.QueueNextJob(); -Console.WriteLine(job == null ? "No pending jobs." : $"Claimed job {job.Id}: {job.Recipe}"); +while (true) +{ + var job = await client.QueueNextJob(); + Console.WriteLine(job == null ? "No pending jobs." : $"Claimed job {job.Id}: {job.Recipe}"); + + if (job != null) + { + GitBinaryClient gitClient = new(); + var uow = new JobUnitOfWork(job, client, gitClient); + await uow.Run(); + } + + await Task.Delay(TimeSpan.FromSeconds(5)); +} \ No newline at end of file diff --git a/src/marlin.agents.shell/marlin.agents.shell.csproj b/src/marlin.agents.shell/marlin.agents.shell.csproj index ff86d63..0278fa0 100644 --- a/src/marlin.agents.shell/marlin.agents.shell.csproj +++ b/src/marlin.agents.shell/marlin.agents.shell.csproj @@ -9,6 +9,8 @@ + + diff --git a/src/marlin.core/Class1.cs b/src/marlin.core/Class1.cs deleted file mode 100644 index ec0b3d9..0000000 --- a/src/marlin.core/Class1.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace marlin.core; - -public class Class1 { } \ No newline at end of file diff --git a/src/marlin.core/contracts/InstanceService.cs b/src/marlin.core/contracts/InstanceService.cs new file mode 100644 index 0000000..73e7229 --- /dev/null +++ b/src/marlin.core/contracts/InstanceService.cs @@ -0,0 +1,43 @@ +namespace marlin.core.contracts; + +using System.Runtime.Serialization; +using System.ServiceModel; +using models; +using ProtoBuf.Grpc; + +[ServiceContract] +public interface IInstanceService +{ + [OperationContract] + Task SendLogEvent(LogEventRequest request, CallContext context = default); + + [OperationContract] + Task SendStatusUpdate(StatusUpdateRequest request, CallContext context = default); +} + +[DataContract] +public class LogEventRequest +{ + [DataMember(Order = 1)] + public DateTime At + { + get => field; + set => field = DateTime.SpecifyKind(value, DateTimeKind.Utc); + } + + [DataMember(Order = 2)] + public string Message { get; set; } + + [DataMember(Order = 3)] + public bool Error { get; set; } + + [DataMember(Order = 4)] + public string StepId { get; set; } +} + +[DataContract] +public class StatusUpdateRequest +{ + [DataMember(Order = 1)] + public JobStatus Status { get; set; } +} \ No newline at end of file diff --git a/src/marlin.core/functional/GitBinaryClient.cs b/src/marlin.core/functional/GitBinaryClient.cs index 3a703da..b2751c0 100644 --- a/src/marlin.core/functional/GitBinaryClient.cs +++ b/src/marlin.core/functional/GitBinaryClient.cs @@ -41,6 +41,13 @@ public class GitBinaryClient : IGitClient await this.RunAsync(authenticate: true, args.ToArray()); } + public async Task Checkout(string repoPath, string remote, string branch, string commit) + { + await this.RunAsync(authenticate: true, "clone", "--revision", branch, remote, repoPath); + // await this.RunAsync(authenticate: true, "clone", "--branch", branch, remote, repoPath); + await this.RunAsync(authenticate: false, "-C", repoPath, "checkout", commit); + } + public async Task ListTree(string repoPath, string hash, string filterPath = ".") { string stdout = await this.RunAsync( @@ -84,7 +91,7 @@ public class GitBinaryClient : IGitClient foreach (string arg in arguments) psi.ArgumentList.Add(arg); - + psi.Environment["GIT_TERMINAL_PROMPT"] = "0"; IAsyncDisposable? credentialScope = authenticate && this._credential != null diff --git a/src/marlin.core/functional/IGitClient.cs b/src/marlin.core/functional/IGitClient.cs index 69cffa3..69476d3 100644 --- a/src/marlin.core/functional/IGitClient.cs +++ b/src/marlin.core/functional/IGitClient.cs @@ -5,6 +5,7 @@ public interface IGitClient Task> GetRemoteRefs(string remote); Task Init(string repoPath); Task FetchTrees(string repoPath, string remote, IEnumerable refs, bool skipBlobs = false); + Task Checkout(string repoPath, string remote, string branch, string commit); Task ListTree(string repoPath, string hash, string filterPath = "."); Task CatFile(string repoPath, string hash, string fileName); } \ No newline at end of file diff --git a/src/marlin.core/marlin.core.csproj b/src/marlin.core/marlin.core.csproj index 5b65fba..044c698 100644 --- a/src/marlin.core/marlin.core.csproj +++ b/src/marlin.core/marlin.core.csproj @@ -7,6 +7,7 @@ + diff --git a/src/marlin.core/models/ISourceSecretStore.cs b/src/marlin.core/models/ISourceSecretStore.cs index 0d5053e..2534d62 100644 --- a/src/marlin.core/models/ISourceSecretStore.cs +++ b/src/marlin.core/models/ISourceSecretStore.cs @@ -6,4 +6,5 @@ using core.models; public interface ISourceSecretStore { Task GetAsync(Guid id, CancellationToken ct); + //Task> ListAsync(CancellationToken ct); } \ No newline at end of file diff --git a/src/marlin.core/models/JobStatus.cs b/src/marlin.core/models/JobStatus.cs new file mode 100644 index 0000000..d606eb0 --- /dev/null +++ b/src/marlin.core/models/JobStatus.cs @@ -0,0 +1,10 @@ +namespace marlin.core.models; + +public enum JobStatus +{ + Pending, + Active, + Cancelled, + Success, + Failed +} \ No newline at end of file diff --git a/src/marlin.core/models/plan/ExecutionCommand.cs b/src/marlin.core/models/plan/ExecutionCommand.cs index 124da63..4bf19c3 100644 --- a/src/marlin.core/models/plan/ExecutionCommand.cs +++ b/src/marlin.core/models/plan/ExecutionCommand.cs @@ -10,6 +10,11 @@ public class ExecutionCommand public required CommandRole Role; public required bool Deferred; public CommandResultSelector? ResultSelector; - + public required FailBehavior FailBehavior; + + public override string ToString() + { + return $"Id={Id}, Name={Name}, Command={Command}, Role={Role}, Deferred={Deferred}, ResultSelector={ResultSelector}, FailBehavior={FailBehavior}"; + } } \ No newline at end of file diff --git a/src/marlin.core/util/CommonPaths.cs b/src/marlin.core/util/CommonPaths.cs new file mode 100644 index 0000000..6659d8b --- /dev/null +++ b/src/marlin.core/util/CommonPaths.cs @@ -0,0 +1,10 @@ +namespace marlin.core.util; + +public static class CommonPaths +{ + public static readonly string Root = Environment.GetEnvironmentVariable("MARLIN_ROOT") + ?? throw new Exception("MARLIN_ROOT is not set"); + + public static readonly string WebRoot = Path.Join(Root, "run"); + public static readonly string AgentRoot = Path.Join(Root, "local-run"); +} \ No newline at end of file diff --git a/src/marlin.web/Controllers/Api/DispatchController.cs b/src/marlin.web/Controllers/Api/DispatchController.cs new file mode 100644 index 0000000..076c4dd --- /dev/null +++ b/src/marlin.web/Controllers/Api/DispatchController.cs @@ -0,0 +1,69 @@ +namespace marlin.web.Controllers.Api; + +using marlin.core.models; +using marlin.core.util; +using marlin.web.Data; +using marlin.web.Models; +using marlin.web.Util; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; + +[Route("api/dispatch")] +[ApiController] +[Authorize(Policy = "Agent")] +public class DispatchController(ApplicationDbContext dbContext) : ControllerBase +{ + [HttpPost("handshake")] + public async Task> Handshake(AgentStatus status, CancellationToken ct) + { + var agentId = this.User.RequireAgentId(); + + var agent = await dbContext.Agents + .FirstAsync(x => x.Id == agentId, ct); + + foreach (var capability in status.CapabilitySet.Items) + agent.Capabilities.Items[capability.Key] = capability.Value; + + await dbContext.SaveChangesAsync(ct); + + return new AgentConfiguration( + Name: agent.Name, + MaxConcurrentJobs: agent.MaxConcurrentJobs + ); + } + + [HttpGet("queue")] + public async Task> Queue(CancellationToken ct) + { + var agentId = this.User.RequireAgentId(); + + var nextJob = await dbContext + .Jobs + .Include(x => x.Source) + .Include(x => x.Branch) + .Include(x => x.Commit) + .Where(x => x.Status == JobStatus.Pending && x.AgentId == null) + .OrderBy(x => x.CreatedAt) + .FirstOrDefaultAsync(ct); + + if (nextJob == null) + return (DispatchJob?)null; + + string claimToken = SecureTokenGenerator.CreateClaimToken(); + nextJob.ClaimToken = claimToken; + nextJob.AgentId = agentId; + + await dbContext.SaveChangesAsync(ct); + + return new DispatchJob( + Id: nextJob.Id, + Token: claimToken, + Vcs: nextJob.Source.UpstreamUrl, + Ref: nextJob.Branch.Name, + Commit: nextJob.Commit.Hash, + Recipe: nextJob.Recipe + ); + } +} + diff --git a/src/marlin.web/Controllers/Api/InstanceGrpcService.cs b/src/marlin.web/Controllers/Api/InstanceGrpcService.cs new file mode 100644 index 0000000..fb0af5a --- /dev/null +++ b/src/marlin.web/Controllers/Api/InstanceGrpcService.cs @@ -0,0 +1,54 @@ +namespace marlin.web.Controllers.Api; + +using core.contracts; +using core.models; +using core.models.plan; +using Data; +using Grpc.Core; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Models; +using ProtoBuf.Grpc; +using Util; + +[Authorize(Policy = "JobScoped")] +public class InstanceGrpcService(ApplicationDbContext dbContext) : IInstanceService +{ + public async Task SendLogEvent(LogEventRequest request, CallContext context = default) + { + var user = context.ServerCallContext!.GetHttpContext().User; + var jobId = user.RequireJobId(); + + dbContext.Logs.Add(new LogEventRecord + { + At = request.At, + Message = request.Message, + Error = request.Error, + StepId = request.StepId, + JobId = jobId + }); + + await dbContext.SaveChangesAsync(context.CancellationToken); + } + + public async Task SendStatusUpdate(StatusUpdateRequest request, CallContext context = default) + { + var user = context.ServerCallContext!.GetHttpContext().User; + var jobId = user.RequireJobId(); + + var job = await dbContext.Jobs + .FirstAsync(x => x.Id == jobId, context.CancellationToken); + + job.Status = request.Status; + + dbContext.JobStatusUpdates.Add(new JobStatusUpdate + { + At = DateTime.UtcNow, + Status = request.Status, + Job = job + }); + + await dbContext.SaveChangesAsync(context.CancellationToken); + } +} diff --git a/src/marlin.web/Controllers/DispatchController.cs b/src/marlin.web/Controllers/DispatchController.cs deleted file mode 100644 index a37d5fa..0000000 --- a/src/marlin.web/Controllers/DispatchController.cs +++ /dev/null @@ -1,71 +0,0 @@ -namespace marlin.web.Controllers; - -using core.models; -using core.models.meta; -using core.util; -using Data; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Mvc; -using Microsoft.EntityFrameworkCore; -using Models; -using Services; -using Util; - -[Route("api/dispatch")] -[ApiController] -[Authorize(Policy = "Agent")] -public class DispatchController(ApplicationDbContext dbContext) : ControllerBase -{ - [HttpPost("handshake")] - public async Task> Handshake(AgentStatus status, CancellationToken ct) - { - var agentId = User.RequireAgentId(); - - var agent = await dbContext.Agents - .FirstAsync(x => x.Id == agentId, ct); - - foreach (var capability in status.CapabilitySet.Items) - agent.Capabilities.Items[capability.Key] = capability.Value; - - await dbContext.SaveChangesAsync(ct); - - return new AgentConfiguration( - Name: agent.Name, - MaxConcurrentJobs: agent.MaxConcurrentJobs - ); - } - - [HttpGet("queue")] - public async Task> Queue(CancellationToken ct) - { - var agentId = User.RequireAgentId(); - - var nextJob = await dbContext - .Jobs - .Include(x => x.Source) - .Include(x => x.Branch) - .Include(x => x.Commit) - .Where(x => x.Status == JobStatus.Pending && x.AgentId == null) - .OrderBy(x => x.CreatedAt) - .FirstOrDefaultAsync(ct); - - if (nextJob == null) - return (DispatchJob?) null; - - string claimToken = SecureTokenGenerator.CreateClaimToken(); - nextJob.ClaimToken = claimToken; - nextJob.AgentId = agentId; - - await dbContext.SaveChangesAsync(ct); - - return new DispatchJob( - Id: nextJob.Id, - Token: claimToken, - Vcs: nextJob.Source.UpstreamUrl, - Ref: nextJob.Branch.Name, - Commit: nextJob.Commit.Hash, - Recipe: nextJob.Recipe - ); - } -} - diff --git a/src/marlin.web/Controllers/JobsController.cs b/src/marlin.web/Controllers/JobsController.cs index 552b56c..9aaf6eb 100644 --- a/src/marlin.web/Controllers/JobsController.cs +++ b/src/marlin.web/Controllers/JobsController.cs @@ -95,6 +95,10 @@ public class JobsController(ApplicationDbContext dbContext) : Controller .Include(x => x.Agent) .Include(x => x.Source) .Include(x => x.StatusUpdates) + .Include(x => x.Logs) + .Include(x => x.Branch) + .Include(x => x.Commit) + .AsSplitQuery() .FirstOrDefaultAsync(x => x.Id == id); if (job == null) diff --git a/src/marlin.web/Data/ApplicationDbContext.cs b/src/marlin.web/Data/ApplicationDbContext.cs index cb6b05b..cbff186 100644 --- a/src/marlin.web/Data/ApplicationDbContext.cs +++ b/src/marlin.web/Data/ApplicationDbContext.cs @@ -26,8 +26,10 @@ public class ApplicationDbContext : DbContext public DbSet SourceBlobs => this.Set(); public DbSet Secrets => this.Set(); public DbSet Jobs => this.Set(); + public DbSet JobStatusUpdates => this.Set(); public DbSet Agents => this.Set(); public DbSet WorkflowSnapshots => this.Set(); + public DbSet Logs => this.Set(); public ApplicationDbContext(DbContextOptions options) : base(options) { } diff --git a/src/marlin.web/Migrations/20261027095603_AddLogEvent.Designer.cs b/src/marlin.web/Migrations/20261027095603_AddLogEvent.Designer.cs new file mode 100644 index 0000000..726076f --- /dev/null +++ b/src/marlin.web/Migrations/20261027095603_AddLogEvent.Designer.cs @@ -0,0 +1,565 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using marlin.web.Data; + +#nullable disable + +namespace marlin.web.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20261027095603_AddLogEvent")] + partial class AddLogEvent + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("marlin.web.Models.Agent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AuthToken") + .IsRequired() + .HasColumnType("text") + .HasColumnName("auth_token"); + + b.Property("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property("BranchId") + .HasColumnType("uuid") + .HasColumnName("branch_id"); + + b.Property("ClaimToken") + .HasColumnType("text") + .HasColumnName("claim_token"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("BranchId") + .HasDatabaseName("ix_jobs_branch_id"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_jobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_update"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_update_job_id"); + + b.ToTable("job_status_update", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("Error") + .HasColumnType("boolean") + .HasColumnName("error"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Message") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message"); + + b.Property("StepId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("step_id"); + + b.HasKey("Id") + .HasName("pk_logs"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_logs_job_id"); + + b.ToTable("logs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("Kind") + .HasColumnType("text") + .HasColumnName("kind"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("MayReveal") + .HasColumnType("boolean") + .HasColumnName("may_reveal"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_secrets"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_secrets_source_id"); + + b.ToTable("secrets", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccessSecretId") + .HasColumnType("uuid") + .HasColumnName("access_secret_id"); + + b.Property("BranchFilter") + .IsRequired() + .HasColumnType("text") + .HasColumnName("branch_filter"); + + b.Property("MainBranch") + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + + b.Property("UpstreamUrl") + .IsRequired() + .HasColumnType("text") + .HasColumnName("upstream_url"); + + b.HasKey("Id") + .HasName("pk_sources"); + + b.ToTable("sources", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_blobs"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_blobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_blobs_source_id"); + + b.ToTable("source_blobs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_branches_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BlobsFetched") + .HasColumnType("boolean") + .HasColumnName("blobs_fetched"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("WorkflowScanned") + .HasColumnType("boolean") + .HasColumnName("workflow_scanned"); + + b.HasKey("Id") + .HasName("pk_source_commits"); + + b.HasAlternateKey("SourceId", "Hash") + .HasName("ak_source_commits_source_id_hash"); + + b.ToTable("source_commits", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("ScannedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("scanned_at"); + + b.Property("Workflow") + .IsRequired() + .HasColumnType("text") + .HasColumnName("workflow"); + + b.HasKey("Id") + .HasName("pk_workflow_snapshots"); + + b.HasIndex("CommitId") + .IsUnique() + .HasDatabaseName("ix_workflow_snapshots_commit_id"); + + b.ToTable("workflow_snapshots", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.HasOne("marlin.web.Models.Agent", "Agent") + .WithMany("Jobs") + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.web.Models.VersionControlledBranch", "Branch") + .WithMany() + .HasForeignKey("BranchId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_branches_branch_id"); + + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany() + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Branch"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => + { + b.HasOne("marlin.web.Models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_update_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.HasOne("marlin.web.Models.Job", "Job") + .WithMany("Logs") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_logs_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_secrets_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany("Blobs") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany("Branches") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_commits_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithOne("WorkflowSnapshot") + .HasForeignKey("marlin.web.Models.WorkflowSnapshot", "CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_workflow_snapshots_source_commits_commit_id"); + + b.Navigation("Commit"); + }); + + modelBuilder.Entity("marlin.web.Models.Agent", b => + { + b.Navigation("Jobs"); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.Navigation("Logs"); + + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.Navigation("Blobs"); + + b.Navigation("Branches"); + + b.Navigation("WorkflowSnapshot"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/marlin.web/Migrations/20261027095603_AddLogEvent.cs b/src/marlin.web/Migrations/20261027095603_AddLogEvent.cs new file mode 100644 index 0000000..b32f68e --- /dev/null +++ b/src/marlin.web/Migrations/20261027095603_AddLogEvent.cs @@ -0,0 +1,67 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace marlin.web.Migrations +{ + /// + public partial class AddLogEvent : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "main_branch", + table: "sources", + type: "text", + nullable: true, + oldClrType: typeof(string), + oldType: "text"); + + migrationBuilder.CreateTable( + name: "logs", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + job_id = table.Column(type: "uuid", nullable: false), + at = table.Column(type: "timestamp with time zone", nullable: false), + message = table.Column(type: "text", nullable: false), + error = table.Column(type: "boolean", nullable: false), + step_id = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_logs", x => x.id); + table.ForeignKey( + name: "fk_logs_jobs_job_id", + column: x => x.job_id, + principalTable: "jobs", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_logs_job_id", + table: "logs", + column: "job_id"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "logs"); + + migrationBuilder.AlterColumn( + name: "main_branch", + table: "sources", + type: "text", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "text", + oldNullable: true); + } + } +} diff --git a/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.Designer.cs b/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.Designer.cs new file mode 100644 index 0000000..5a199ba --- /dev/null +++ b/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.Designer.cs @@ -0,0 +1,565 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using marlin.web.Data; + +#nullable disable + +namespace marlin.web.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20261027130151_JobStatusUpdatesTable")] + partial class JobStatusUpdatesTable + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("marlin.web.Models.Agent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AuthToken") + .IsRequired() + .HasColumnType("text") + .HasColumnName("auth_token"); + + b.Property("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property("BranchId") + .HasColumnType("uuid") + .HasColumnName("branch_id"); + + b.Property("ClaimToken") + .HasColumnType("text") + .HasColumnName("claim_token"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("BranchId") + .HasDatabaseName("ix_jobs_branch_id"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_jobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_updates"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_updates_job_id"); + + b.ToTable("job_status_updates", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("Error") + .HasColumnType("boolean") + .HasColumnName("error"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Message") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message"); + + b.Property("StepId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("step_id"); + + b.HasKey("Id") + .HasName("pk_logs"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_logs_job_id"); + + b.ToTable("logs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("Kind") + .HasColumnType("text") + .HasColumnName("kind"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("MayReveal") + .HasColumnType("boolean") + .HasColumnName("may_reveal"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_secrets"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_secrets_source_id"); + + b.ToTable("secrets", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccessSecretId") + .HasColumnType("uuid") + .HasColumnName("access_secret_id"); + + b.Property("BranchFilter") + .IsRequired() + .HasColumnType("text") + .HasColumnName("branch_filter"); + + b.Property("MainBranch") + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + + b.Property("UpstreamUrl") + .IsRequired() + .HasColumnType("text") + .HasColumnName("upstream_url"); + + b.HasKey("Id") + .HasName("pk_sources"); + + b.ToTable("sources", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_blobs"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_blobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_blobs_source_id"); + + b.ToTable("source_blobs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_branches_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BlobsFetched") + .HasColumnType("boolean") + .HasColumnName("blobs_fetched"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("WorkflowScanned") + .HasColumnType("boolean") + .HasColumnName("workflow_scanned"); + + b.HasKey("Id") + .HasName("pk_source_commits"); + + b.HasAlternateKey("SourceId", "Hash") + .HasName("ak_source_commits_source_id_hash"); + + b.ToTable("source_commits", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("ScannedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("scanned_at"); + + b.Property("Workflow") + .IsRequired() + .HasColumnType("text") + .HasColumnName("workflow"); + + b.HasKey("Id") + .HasName("pk_workflow_snapshots"); + + b.HasIndex("CommitId") + .IsUnique() + .HasDatabaseName("ix_workflow_snapshots_commit_id"); + + b.ToTable("workflow_snapshots", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.HasOne("marlin.web.Models.Agent", "Agent") + .WithMany("Jobs") + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.web.Models.VersionControlledBranch", "Branch") + .WithMany() + .HasForeignKey("BranchId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_branches_branch_id"); + + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany() + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Branch"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => + { + b.HasOne("marlin.web.Models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_updates_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.HasOne("marlin.web.Models.Job", "Job") + .WithMany("Logs") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_logs_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_secrets_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany("Blobs") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany("Branches") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_commits_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithOne("WorkflowSnapshot") + .HasForeignKey("marlin.web.Models.WorkflowSnapshot", "CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_workflow_snapshots_source_commits_commit_id"); + + b.Navigation("Commit"); + }); + + modelBuilder.Entity("marlin.web.Models.Agent", b => + { + b.Navigation("Jobs"); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.Navigation("Logs"); + + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.Navigation("Blobs"); + + b.Navigation("Branches"); + + b.Navigation("WorkflowSnapshot"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.cs b/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.cs new file mode 100644 index 0000000..9cf6546 --- /dev/null +++ b/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.cs @@ -0,0 +1,78 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace marlin.web.Migrations +{ + /// + public partial class JobStatusUpdatesTable : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "fk_job_status_update_jobs_job_id", + table: "job_status_update"); + + migrationBuilder.DropPrimaryKey( + name: "pk_job_status_update", + table: "job_status_update"); + + migrationBuilder.RenameTable( + name: "job_status_update", + newName: "job_status_updates"); + + migrationBuilder.RenameIndex( + name: "ix_job_status_update_job_id", + table: "job_status_updates", + newName: "ix_job_status_updates_job_id"); + + migrationBuilder.AddPrimaryKey( + name: "pk_job_status_updates", + table: "job_status_updates", + column: "id"); + + migrationBuilder.AddForeignKey( + name: "fk_job_status_updates_jobs_job_id", + table: "job_status_updates", + column: "job_id", + principalTable: "jobs", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "fk_job_status_updates_jobs_job_id", + table: "job_status_updates"); + + migrationBuilder.DropPrimaryKey( + name: "pk_job_status_updates", + table: "job_status_updates"); + + migrationBuilder.RenameTable( + name: "job_status_updates", + newName: "job_status_update"); + + migrationBuilder.RenameIndex( + name: "ix_job_status_updates_job_id", + table: "job_status_update", + newName: "ix_job_status_update_job_id"); + + migrationBuilder.AddPrimaryKey( + name: "pk_job_status_update", + table: "job_status_update", + column: "id"); + + migrationBuilder.AddForeignKey( + name: "fk_job_status_update_jobs_job_id", + table: "job_status_update", + column: "job_id", + principalTable: "jobs", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + } + } +} diff --git a/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs b/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs index 411706b..da08d88 100644 --- a/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs @@ -141,12 +141,50 @@ namespace marlin.web.Migrations .HasColumnName("status"); b.HasKey("Id") - .HasName("pk_job_status_update"); + .HasName("pk_job_status_updates"); b.HasIndex("JobId") - .HasDatabaseName("ix_job_status_update_job_id"); + .HasDatabaseName("ix_job_status_updates_job_id"); - b.ToTable("job_status_update", (string)null); + b.ToTable("job_status_updates", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("Error") + .HasColumnType("boolean") + .HasColumnName("error"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Message") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message"); + + b.Property("StepId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("step_id"); + + b.HasKey("Id") + .HasName("pk_logs"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_logs_job_id"); + + b.ToTable("logs", (string)null); }); modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => @@ -204,7 +242,6 @@ namespace marlin.web.Migrations .HasColumnName("branch_filter"); b.Property("MainBranch") - .IsRequired() .HasColumnType("text") .HasColumnName("main_branch"); @@ -399,7 +436,19 @@ namespace marlin.web.Migrations .HasForeignKey("JobId") .OnDelete(DeleteBehavior.Cascade) .IsRequired() - .HasConstraintName("fk_job_status_update_jobs_job_id"); + .HasConstraintName("fk_job_status_updates_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.HasOne("marlin.web.Models.Job", "Job") + .WithMany("Logs") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_logs_jobs_job_id"); b.Navigation("Job"); }); @@ -489,6 +538,8 @@ namespace marlin.web.Migrations modelBuilder.Entity("marlin.web.Models.Job", b => { + b.Navigation("Logs"); + b.Navigation("StatusUpdates"); }); diff --git a/src/marlin.web/Models/Job.cs b/src/marlin.web/Models/Job.cs index bd60773..f7b6278 100644 --- a/src/marlin.web/Models/Job.cs +++ b/src/marlin.web/Models/Job.cs @@ -1,5 +1,7 @@ namespace marlin.web.Models; +using core.models; + public class Job { public Guid Id { get; set; } @@ -25,6 +27,8 @@ public class Job public string Recipe { get; set; } // plan + + public ICollection Logs { get; set; } } public class JobStatusUpdate @@ -36,13 +40,4 @@ public class JobStatusUpdate public DateTime At { get; set; } public JobStatus Status { get; set; } -} - -public enum JobStatus -{ - Pending, - Active, - Cancelled, - Success, - Failed } \ No newline at end of file diff --git a/src/marlin.web/Models/LogEventRecord.cs b/src/marlin.web/Models/LogEventRecord.cs new file mode 100644 index 0000000..1a916d8 --- /dev/null +++ b/src/marlin.web/Models/LogEventRecord.cs @@ -0,0 +1,16 @@ +namespace marlin.web.Models; + +using core.models; + +public class LogEventRecord +{ + public Guid Id { get; set; } + + public Guid JobId { get; set; } + public Job? Job { get; set; } + + public DateTime At { get; set; } + public string Message { get; set; } + public bool Error { get; set; } + public string StepId { get; set; } +} \ No newline at end of file diff --git a/src/marlin.web/Program.cs b/src/marlin.web/Program.cs index 700e696..8dea3fe 100644 --- a/src/marlin.web/Program.cs +++ b/src/marlin.web/Program.cs @@ -1,12 +1,16 @@ +using marlin.web.Controllers.Api; using marlin.web.Data; using marlin.web.Models; using marlin.web.Services; using Microsoft.EntityFrameworkCore; +using ProtoBuf.Grpc.Server; DotNetEnv.Env.NoClobber().TraversePath().Load(); var builder = WebApplication.CreateBuilder(args); +builder.Services.AddCodeFirstGrpc(); + builder.Services.AddDbContext(options => options .UseNpgsql(builder.Configuration.GetConnectionString("Master")) .UseSnakeCaseNamingConvention()); @@ -20,7 +24,7 @@ builder.Services builder.Services.AddAuthorization(options => { options.AddPolicy("Agent", p => p.RequireClaim("agent_id")); - options.AddPolicy("JobScoped", p => p.RequireClaim("job_id")); + options.AddPolicy("JobScoped", p => p.RequireClaim("agent_id").RequireClaim("job_id")); }); builder.Services.AddHostedService(); @@ -49,6 +53,8 @@ app.UseRouting(); app.UseAuthentication(); app.UseAuthorization(); +app.MapGrpcService(); + app.MapStaticAssets(); app diff --git a/src/marlin.web/Services/VersionControlSyncService.cs b/src/marlin.web/Services/VersionControlSyncService.cs index 8f2046e..4509745 100644 --- a/src/marlin.web/Services/VersionControlSyncService.cs +++ b/src/marlin.web/Services/VersionControlSyncService.cs @@ -115,7 +115,7 @@ public class VersionControlSyncService : BackgroundService private async Task GetLocalPath(IGitClient client, VersionControlSource source) { - var dir = Path.Join(Environment.CurrentDirectory, "run", "sources", source.Id.ToString("N")); + var dir = Path.Join(CommonPaths.WebRoot, "sources", source.Id.ToString("N")); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); diff --git a/src/marlin.web/Views/Jobs/Details.cshtml b/src/marlin.web/Views/Jobs/Details.cshtml index ebba94d..9b27fe3 100644 --- a/src/marlin.web/Views/Jobs/Details.cshtml +++ b/src/marlin.web/Views/Jobs/Details.cshtml @@ -2,4 +2,27 @@ @Model.Id -Back \ No newline at end of file +@Model.Branch.Name + +
+
Id
+
@Model.Id
+ +
Branch
+
@Model.Branch.Name
+ +
Commit
+
@Model.Commit.Hash
+ +
Status
+
@Model.Status
+
+ +
    + @foreach (var line in Model.Logs.OrderBy(x=>x.At)) + { +
  • [@line.At.ToString("HH:mm:ss")] [@line.StepId] @line.Message
    • + } +
+ +Back diff --git a/src/marlin.web/marlin.web.csproj b/src/marlin.web/marlin.web.csproj index 26c8ea3..b931039 100644 --- a/src/marlin.web/marlin.web.csproj +++ b/src/marlin.web/marlin.web.csproj @@ -15,6 +15,8 @@ all + + -- cgit v1.3.1