diff options
Diffstat (limited to 'src')
29 files changed, 1762 insertions, 39 deletions
diff --git a/src/marlin.agents.shell/CommandRunner.cs b/src/marlin.agents.shell/CommandRunner.cs new file mode 100644 index 0000000..d4fcbb3 --- /dev/null +++ b/src/marlin.agents.shell/CommandRunner.cs @@ -0,0 +1,38 @@ +using System.Diagnostics; + +public static class CommandRunner +{ + public static async Task<int> RunAsync(string command, Action<string> onStdout, Action<string> onStderr) + { + var psi = new ProcessStartInfo + { + FileName = GetShell(), + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + var process = new Process { StartInfo = psi }; + process.OutputDataReceived += (_, e) => { if (e.Data != null) onStdout(e.Data); }; + process.ErrorDataReceived += (_, e) => { if (e.Data != null) onStderr(e.Data); }; + + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + // Write the command to stdin and close it + await process.StandardInput.WriteLineAsync(command); + await process.StandardInput.WriteLineAsync("exit $?"); // Exit with the command's exit code + process.StandardInput.Close(); + + await process.WaitForExitAsync(); + return process.ExitCode; + } + + private static string GetShell() + { + return OperatingSystem.IsWindows() ? "cmd.exe" : "/bin/bash"; + } +}
\ No newline at end of file diff --git a/src/marlin.agents.shell/JobUnitOfWork.cs b/src/marlin.agents.shell/JobUnitOfWork.cs new file mode 100644 index 0000000..85df806 --- /dev/null +++ b/src/marlin.agents.shell/JobUnitOfWork.cs @@ -0,0 +1,130 @@ +namespace marlin.agents.shell; + +using System.Threading.Channels; +using core.contracts; +using core.functional; +using core.models; +using core.models.meta; +using core.models.plan; +using core.util; +using Grpc.Net.Client; +using ProtoBuf.Grpc.Client; + +public class JobUnitOfWork +{ + private static string _jobInstanceDir = Path.Combine(CommonPaths.AgentRoot, "instance"); + + private readonly MarlinApiClient _apiClient; + private readonly IGitClient _gitClient; + private readonly DispatchJob _job; + private readonly string _checkoutPath; + + private GrpcChannel _channel; + private IInstanceService _instanceService; + + private Channel<LogEventRequest> _logQueue; + + public JobUnitOfWork(DispatchJob job, MarlinApiClient apiClient, IGitClient gitClient) + { + this._job = job; + this._apiClient = apiClient; + this._gitClient = gitClient; + + this._checkoutPath = Path.Combine(_jobInstanceDir, job.Id.ToString("N")); + } + + public async Task Run() + { + this._channel = this._apiClient.CreateJobScopedRpc(this._job); + this._instanceService = this._channel.CreateGrpcService<IInstanceService>(); + this._logQueue = Channel.CreateUnbounded<LogEventRequest>(); + + var drainTask = Task.Run(this.DrainLogEvents); + + await StatusUpdate(JobStatus.Active); + + await this.CheckoutSource(); + var plan = await this.BuildPlan(); + var success = await this.ExecutePlan(plan); + + await StatusUpdate(success ? JobStatus.Success : JobStatus.Failed); + + this._logQueue.Writer.Complete(); + await drainTask; + } + + private async Task CheckoutSource() + { + Directory.CreateDirectory(_jobInstanceDir); + if (Path.Exists(this._checkoutPath)) + Directory.Delete(this._checkoutPath, true); + + await this._gitClient.Checkout(this._checkoutPath, this._job.Vcs, this._job.Ref, this._job.Commit); + } + + private async Task DrainLogEvents() + { + await foreach (var evt in this._logQueue.Reader.ReadAllAsync()) + await this._instanceService.SendLogEvent(evt); + } + + private async Task<ExecutionPlan> BuildPlan() + { + WorkflowBuilder builder = new(); + builder.LoadFromDirectory(Path.Join(this._checkoutPath, ".marlin")); + + Workflow workflow = builder.Build(); + Recipe recipe = workflow.Recipes[this._job.Recipe]; + ExecutionPlan plan = ExecutionPlan.Create(workflow, recipe); + + return plan; + } + + private async Task<bool> ExecutePlan(ExecutionPlan plan) + { + Dictionary<string, CommandResult> results = new(); + + foreach (var component in plan.Components) + { + if (component.ResultSelector != null) + { + if (!results.ContainsKey(component.ResultSelector.Command.Id)) + // prerequisite step did not run + continue; + + var storedResult = results[component.ResultSelector.Command.Id]; + if (!storedResult.Matches(component.ResultSelector.Result)) + // prerequisite step returned incorrect error code + continue; + } + + Console.WriteLine(component); + Task<int> run = CommandRunner.RunAsync( + component.Command, + o => Log(o, false, component), + o => Log(o, true, component)); + + var result = await run; + results[component.Id] = result == 0 ? CommandResult.Success : CommandResult.Failure; + } + + return results.All(x => x.Value == CommandResult.Success); + } + + private void Log(string message, bool error, ExecutionCommand component) + { + this._logQueue.Writer.TryWrite( + new LogEventRequest + { + At = DateTime.UtcNow, + Error = error, + Message = message, + StepId = component.Id + }); + } + + private async Task StatusUpdate(JobStatus status) + { + await this._instanceService.SendStatusUpdate(new StatusUpdateRequest { Status = status }); + } +}
\ No newline at end of file diff --git a/src/marlin.agents.shell/MarlinApiClient.cs b/src/marlin.agents.shell/MarlinApiClient.cs index aa4f013..900101c 100644 --- a/src/marlin.agents.shell/MarlinApiClient.cs +++ b/src/marlin.agents.shell/MarlinApiClient.cs @@ -1,16 +1,28 @@ namespace marlin.agents.shell; using System.Net.Http.Json; +using core.contracts; using core.models; +using Grpc.Core; +using Grpc.Net.Client; +using ProtoBuf.Grpc.Client; public class MarlinApiClient { + private readonly string _baseUrl; + private readonly Guid _agentId; + private readonly string _agentToken; + private readonly HttpClient _http; public MarlinApiClient(string baseUrl, Guid agentId, string agentToken) { - _http = new HttpClient { BaseAddress = new Uri(baseUrl) }; - _http.DefaultRequestHeaders.Add("Agent", $"{agentId} {agentToken}"); + this._baseUrl = baseUrl; + this._agentId = agentId; + this._agentToken = agentToken; + + _http = new HttpClient { BaseAddress = new Uri(this._baseUrl) }; + _http.DefaultRequestHeaders.Add("Agent", $"{this._agentId} {this._agentToken}"); } public static MarlinApiClient FromEnvironment() @@ -35,4 +47,21 @@ public class MarlinApiClient public async Task<DispatchJob?> QueueNextJob(CancellationToken ct = default) => await _http.GetFromJsonAsync<DispatchJob?>("api/dispatch/queue", ct); + + public GrpcChannel CreateJobScopedRpc(DispatchJob job) + { + var credentials = CallCredentials.FromInterceptor( + async (_, metadata) => + { + metadata.Add("Agent", $"{this._agentId} {this._agentToken}"); + metadata.Add("Job", $"{job.Id} {job.Token}"); + }); + + var channel = GrpcChannel.ForAddress(this._baseUrl, new GrpcChannelOptions + { + Credentials = ChannelCredentials.Create(new SslCredentials(), credentials) + }); + + return channel; + } } diff --git a/src/marlin.agents.shell/Program.cs b/src/marlin.agents.shell/Program.cs index 02a0bc9..75ea8ad 100644 --- a/src/marlin.agents.shell/Program.cs +++ b/src/marlin.agents.shell/Program.cs @@ -1,9 +1,20 @@ using System.Runtime.InteropServices; +using System.Threading.Channels; +using Grpc.Net.Client; using marlin.agents.shell; +using marlin.core.contracts; +using marlin.core.functional; using marlin.core.models; +using marlin.core.models.meta; +using marlin.core.models.plan; +using marlin.core.util; +using ProtoBuf.Grpc.Client; DotNetEnv.Env.NoClobber().TraversePath().Load(); +var jobInstanceDir = Path.Combine(CommonPaths.AgentRoot, "instance"); +Directory.CreateDirectory(jobInstanceDir); + var client = MarlinApiClient.FromEnvironment(); var capabilities = new CapabilitySet(); @@ -16,5 +27,17 @@ AgentConfiguration config = await client.Handshake(status); Console.WriteLine($"IAM: {config.Name}"); Console.WriteLine($"MCJ: {config.MaxConcurrentJobs}"); -var job = await client.QueueNextJob(); -Console.WriteLine(job == null ? "No pending jobs." : $"Claimed job {job.Id}: {job.Recipe}"); +while (true) +{ + var job = await client.QueueNextJob(); + Console.WriteLine(job == null ? "No pending jobs." : $"Claimed job {job.Id}: {job.Recipe}"); + + if (job != null) + { + GitBinaryClient gitClient = new(); + var uow = new JobUnitOfWork(job, client, gitClient); + await uow.Run(); + } + + await Task.Delay(TimeSpan.FromSeconds(5)); +}
\ No newline at end of file diff --git a/src/marlin.agents.shell/marlin.agents.shell.csproj b/src/marlin.agents.shell/marlin.agents.shell.csproj index ff86d63..0278fa0 100644 --- a/src/marlin.agents.shell/marlin.agents.shell.csproj +++ b/src/marlin.agents.shell/marlin.agents.shell.csproj @@ -9,6 +9,8 @@ <ItemGroup> <PackageReference Include="DotNetEnv" Version="3.2.0" /> + <PackageReference Include="Grpc.Net.Client" Version="2.84.0" /> + <PackageReference Include="protobuf-net.Grpc" Version="1.3.14" /> </ItemGroup> <ItemGroup> diff --git a/src/marlin.core/Class1.cs b/src/marlin.core/Class1.cs deleted file mode 100644 index ec0b3d9..0000000 --- a/src/marlin.core/Class1.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace marlin.core; - -public class Class1 { }
\ No newline at end of file diff --git a/src/marlin.core/contracts/InstanceService.cs b/src/marlin.core/contracts/InstanceService.cs new file mode 100644 index 0000000..73e7229 --- /dev/null +++ b/src/marlin.core/contracts/InstanceService.cs @@ -0,0 +1,43 @@ +namespace marlin.core.contracts; + +using System.Runtime.Serialization; +using System.ServiceModel; +using models; +using ProtoBuf.Grpc; + +[ServiceContract] +public interface IInstanceService +{ + [OperationContract] + Task SendLogEvent(LogEventRequest request, CallContext context = default); + + [OperationContract] + Task SendStatusUpdate(StatusUpdateRequest request, CallContext context = default); +} + +[DataContract] +public class LogEventRequest +{ + [DataMember(Order = 1)] + public DateTime At + { + get => field; + set => field = DateTime.SpecifyKind(value, DateTimeKind.Utc); + } + + [DataMember(Order = 2)] + public string Message { get; set; } + + [DataMember(Order = 3)] + public bool Error { get; set; } + + [DataMember(Order = 4)] + public string StepId { get; set; } +} + +[DataContract] +public class StatusUpdateRequest +{ + [DataMember(Order = 1)] + public JobStatus Status { get; set; } +}
\ No newline at end of file diff --git a/src/marlin.core/functional/GitBinaryClient.cs b/src/marlin.core/functional/GitBinaryClient.cs index 3a703da..b2751c0 100644 --- a/src/marlin.core/functional/GitBinaryClient.cs +++ b/src/marlin.core/functional/GitBinaryClient.cs @@ -41,6 +41,13 @@ public class GitBinaryClient : IGitClient await this.RunAsync(authenticate: true, args.ToArray()); } + public async Task Checkout(string repoPath, string remote, string branch, string commit) + { + await this.RunAsync(authenticate: true, "clone", "--revision", branch, remote, repoPath); + // await this.RunAsync(authenticate: true, "clone", "--branch", branch, remote, repoPath); + await this.RunAsync(authenticate: false, "-C", repoPath, "checkout", commit); + } + public async Task<string[]> ListTree(string repoPath, string hash, string filterPath = ".") { string stdout = await this.RunAsync( @@ -84,7 +91,7 @@ public class GitBinaryClient : IGitClient foreach (string arg in arguments) psi.ArgumentList.Add(arg); - + psi.Environment["GIT_TERMINAL_PROMPT"] = "0"; IAsyncDisposable? credentialScope = authenticate && this._credential != null diff --git a/src/marlin.core/functional/IGitClient.cs b/src/marlin.core/functional/IGitClient.cs index 69cffa3..69476d3 100644 --- a/src/marlin.core/functional/IGitClient.cs +++ b/src/marlin.core/functional/IGitClient.cs @@ -5,6 +5,7 @@ public interface IGitClient Task<Dictionary<string, string>> GetRemoteRefs(string remote); Task Init(string repoPath); Task FetchTrees(string repoPath, string remote, IEnumerable<string> refs, bool skipBlobs = false); + Task Checkout(string repoPath, string remote, string branch, string commit); Task<string[]> ListTree(string repoPath, string hash, string filterPath = "."); Task<byte[]> CatFile(string repoPath, string hash, string fileName); }
\ No newline at end of file diff --git a/src/marlin.core/marlin.core.csproj b/src/marlin.core/marlin.core.csproj index 5b65fba..044c698 100644 --- a/src/marlin.core/marlin.core.csproj +++ b/src/marlin.core/marlin.core.csproj @@ -7,6 +7,7 @@ </PropertyGroup> <ItemGroup> + <PackageReference Include="protobuf-net.Grpc" Version="1.3.14" /> <PackageReference Include="YamlDotNet" Version="18.1.0" /> </ItemGroup> diff --git a/src/marlin.core/models/ISourceSecretStore.cs b/src/marlin.core/models/ISourceSecretStore.cs index 0d5053e..2534d62 100644 --- a/src/marlin.core/models/ISourceSecretStore.cs +++ b/src/marlin.core/models/ISourceSecretStore.cs @@ -6,4 +6,5 @@ using core.models; public interface ISourceSecretStore { Task<ISecret?> GetAsync(Guid id, CancellationToken ct); + //Task<IEnumerable<ISecret>> ListAsync(CancellationToken ct); }
\ No newline at end of file diff --git a/src/marlin.core/models/JobStatus.cs b/src/marlin.core/models/JobStatus.cs new file mode 100644 index 0000000..d606eb0 --- /dev/null +++ b/src/marlin.core/models/JobStatus.cs @@ -0,0 +1,10 @@ +namespace marlin.core.models; + +public enum JobStatus +{ + Pending, + Active, + Cancelled, + Success, + Failed +}
\ No newline at end of file diff --git a/src/marlin.core/models/plan/ExecutionCommand.cs b/src/marlin.core/models/plan/ExecutionCommand.cs index 124da63..4bf19c3 100644 --- a/src/marlin.core/models/plan/ExecutionCommand.cs +++ b/src/marlin.core/models/plan/ExecutionCommand.cs @@ -10,6 +10,11 @@ public class ExecutionCommand public required CommandRole Role; public required bool Deferred; public CommandResultSelector? ResultSelector; - + public required FailBehavior FailBehavior; + + public override string ToString() + { + return $"Id={Id}, Name={Name}, Command={Command}, Role={Role}, Deferred={Deferred}, ResultSelector={ResultSelector}, FailBehavior={FailBehavior}"; + } }
\ No newline at end of file diff --git a/src/marlin.core/util/CommonPaths.cs b/src/marlin.core/util/CommonPaths.cs new file mode 100644 index 0000000..6659d8b --- /dev/null +++ b/src/marlin.core/util/CommonPaths.cs @@ -0,0 +1,10 @@ +namespace marlin.core.util; + +public static class CommonPaths +{ + public static readonly string Root = Environment.GetEnvironmentVariable("MARLIN_ROOT") + ?? throw new Exception("MARLIN_ROOT is not set"); + + public static readonly string WebRoot = Path.Join(Root, "run"); + public static readonly string AgentRoot = Path.Join(Root, "local-run"); +}
\ No newline at end of file diff --git a/src/marlin.web/Controllers/DispatchController.cs b/src/marlin.web/Controllers/Api/DispatchController.cs index a37d5fa..076c4dd 100644 --- a/src/marlin.web/Controllers/DispatchController.cs +++ b/src/marlin.web/Controllers/Api/DispatchController.cs @@ -1,15 +1,13 @@ -namespace marlin.web.Controllers; +namespace marlin.web.Controllers.Api; -using core.models; -using core.models.meta; -using core.util; -using Data; +using marlin.core.models; +using marlin.core.util; +using marlin.web.Data; +using marlin.web.Models; +using marlin.web.Util; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -using Models; -using Services; -using Util; [Route("api/dispatch")] [ApiController] @@ -19,7 +17,7 @@ public class DispatchController(ApplicationDbContext dbContext) : ControllerBase [HttpPost("handshake")] public async Task<ActionResult<AgentConfiguration>> Handshake(AgentStatus status, CancellationToken ct) { - var agentId = User.RequireAgentId(); + var agentId = this.User.RequireAgentId(); var agent = await dbContext.Agents .FirstAsync(x => x.Id == agentId, ct); @@ -34,11 +32,11 @@ public class DispatchController(ApplicationDbContext dbContext) : ControllerBase MaxConcurrentJobs: agent.MaxConcurrentJobs ); } - + [HttpGet("queue")] public async Task<ActionResult<DispatchJob?>> Queue(CancellationToken ct) { - var agentId = User.RequireAgentId(); + var agentId = this.User.RequireAgentId(); var nextJob = await dbContext .Jobs @@ -50,7 +48,7 @@ public class DispatchController(ApplicationDbContext dbContext) : ControllerBase .FirstOrDefaultAsync(ct); if (nextJob == null) - return (DispatchJob?) null; + return (DispatchJob?)null; string claimToken = SecureTokenGenerator.CreateClaimToken(); nextJob.ClaimToken = claimToken; @@ -66,6 +64,6 @@ public class DispatchController(ApplicationDbContext dbContext) : ControllerBase Commit: nextJob.Commit.Hash, Recipe: nextJob.Recipe ); - } + } } diff --git a/src/marlin.web/Controllers/Api/InstanceGrpcService.cs b/src/marlin.web/Controllers/Api/InstanceGrpcService.cs new file mode 100644 index 0000000..fb0af5a --- /dev/null +++ b/src/marlin.web/Controllers/Api/InstanceGrpcService.cs @@ -0,0 +1,54 @@ +namespace marlin.web.Controllers.Api; + +using core.contracts; +using core.models; +using core.models.plan; +using Data; +using Grpc.Core; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Models; +using ProtoBuf.Grpc; +using Util; + +[Authorize(Policy = "JobScoped")] +public class InstanceGrpcService(ApplicationDbContext dbContext) : IInstanceService +{ + public async Task SendLogEvent(LogEventRequest request, CallContext context = default) + { + var user = context.ServerCallContext!.GetHttpContext().User; + var jobId = user.RequireJobId(); + + dbContext.Logs.Add(new LogEventRecord + { + At = request.At, + Message = request.Message, + Error = request.Error, + StepId = request.StepId, + JobId = jobId + }); + + await dbContext.SaveChangesAsync(context.CancellationToken); + } + + public async Task SendStatusUpdate(StatusUpdateRequest request, CallContext context = default) + { + var user = context.ServerCallContext!.GetHttpContext().User; + var jobId = user.RequireJobId(); + + var job = await dbContext.Jobs + .FirstAsync(x => x.Id == jobId, context.CancellationToken); + + job.Status = request.Status; + + dbContext.JobStatusUpdates.Add(new JobStatusUpdate + { + At = DateTime.UtcNow, + Status = request.Status, + Job = job + }); + + await dbContext.SaveChangesAsync(context.CancellationToken); + } +} diff --git a/src/marlin.web/Controllers/JobsController.cs b/src/marlin.web/Controllers/JobsController.cs index 552b56c..9aaf6eb 100644 --- a/src/marlin.web/Controllers/JobsController.cs +++ b/src/marlin.web/Controllers/JobsController.cs @@ -95,6 +95,10 @@ public class JobsController(ApplicationDbContext dbContext) : Controller .Include(x => x.Agent) .Include(x => x.Source) .Include(x => x.StatusUpdates) + .Include(x => x.Logs) + .Include(x => x.Branch) + .Include(x => x.Commit) + .AsSplitQuery() .FirstOrDefaultAsync(x => x.Id == id); if (job == null) diff --git a/src/marlin.web/Data/ApplicationDbContext.cs b/src/marlin.web/Data/ApplicationDbContext.cs index cb6b05b..cbff186 100644 --- a/src/marlin.web/Data/ApplicationDbContext.cs +++ b/src/marlin.web/Data/ApplicationDbContext.cs @@ -26,8 +26,10 @@ public class ApplicationDbContext : DbContext public DbSet<VersionControlledBlob> SourceBlobs => this.Set<VersionControlledBlob>(); public DbSet<VersionControlSecret> Secrets => this.Set<VersionControlSecret>(); public DbSet<Job> Jobs => this.Set<Job>(); + public DbSet<JobStatusUpdate> JobStatusUpdates => this.Set<JobStatusUpdate>(); public DbSet<Agent> Agents => this.Set<Agent>(); public DbSet<WorkflowSnapshot> WorkflowSnapshots => this.Set<WorkflowSnapshot>(); + public DbSet<LogEventRecord> Logs => this.Set<LogEventRecord>(); public ApplicationDbContext(DbContextOptions<ApplicationDbContext> options) : base(options) { } diff --git a/src/marlin.web/Migrations/20261027095603_AddLogEvent.Designer.cs b/src/marlin.web/Migrations/20261027095603_AddLogEvent.Designer.cs new file mode 100644 index 0000000..726076f --- /dev/null +++ b/src/marlin.web/Migrations/20261027095603_AddLogEvent.Designer.cs @@ -0,0 +1,565 @@ +// <auto-generated /> +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using marlin.web.Data; + +#nullable disable + +namespace marlin.web.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20261027095603_AddLogEvent")] + partial class AddLogEvent + { + /// <inheritdoc /> + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("marlin.web.Models.Agent", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<string>("AuthToken") + .IsRequired() + .HasColumnType("text") + .HasColumnName("auth_token"); + + b.Property<string>("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property<DateTime>("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property<int>("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property<string>("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<Guid?>("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property<Guid>("BranchId") + .HasColumnType("uuid") + .HasColumnName("branch_id"); + + b.Property<string>("ClaimToken") + .HasColumnType("text") + .HasColumnName("claim_token"); + + b.Property<Guid>("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property<DateTime>("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property<string>("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property<string>("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property<Guid>("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property<int>("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("BranchId") + .HasDatabaseName("ix_jobs_branch_id"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_jobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<DateTime>("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property<Guid>("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property<int>("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_update"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_update_job_id"); + + b.ToTable("job_status_update", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<DateTime>("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property<bool>("Error") + .HasColumnType("boolean") + .HasColumnName("error"); + + b.Property<Guid>("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property<string>("Message") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message"); + + b.Property<string>("StepId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("step_id"); + + b.HasKey("Id") + .HasName("pk_logs"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_logs_job_id"); + + b.ToTable("logs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<string>("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property<string>("Kind") + .HasColumnType("text") + .HasColumnName("kind"); + + b.Property<string>("Label") + .IsRequired() + .HasColumnType("text") + .HasColumnName("label"); + + b.Property<bool>("MayReveal") + .HasColumnType("boolean") + .HasColumnName("may_reveal"); + + b.Property<Guid>("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_secrets"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_secrets_source_id"); + + b.ToTable("secrets", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<Guid?>("AccessSecretId") + .HasColumnType("uuid") + .HasColumnName("access_secret_id"); + + b.Property<string>("BranchFilter") + .IsRequired() + .HasColumnType("text") + .HasColumnName("branch_filter"); + + b.Property<string>("MainBranch") + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property<int>("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + + b.Property<string>("UpstreamUrl") + .IsRequired() + .HasColumnType("text") + .HasColumnName("upstream_url"); + + b.HasKey("Id") + .HasName("pk_sources"); + + b.ToTable("sources", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<Guid>("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property<byte[]>("Content") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.Property<string>("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property<Guid>("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_blobs"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_blobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_blobs_source_id"); + + b.ToTable("source_blobs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<Guid>("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property<string>("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property<Guid>("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_branches_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<bool>("BlobsFetched") + .HasColumnType("boolean") + .HasColumnName("blobs_fetched"); + + b.Property<string>("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property<Guid>("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property<bool>("WorkflowScanned") + .HasColumnType("boolean") + .HasColumnName("workflow_scanned"); + + b.HasKey("Id") + .HasName("pk_source_commits"); + + b.HasAlternateKey("SourceId", "Hash") + .HasName("ak_source_commits_source_id_hash"); + + b.ToTable("source_commits", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<Guid>("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property<DateTime>("ScannedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("scanned_at"); + + b.Property<string>("Workflow") + .IsRequired() + .HasColumnType("text") + .HasColumnName("workflow"); + + b.HasKey("Id") + .HasName("pk_workflow_snapshots"); + + b.HasIndex("CommitId") + .IsUnique() + .HasDatabaseName("ix_workflow_snapshots_commit_id"); + + b.ToTable("workflow_snapshots", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.HasOne("marlin.web.Models.Agent", "Agent") + .WithMany("Jobs") + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.web.Models.VersionControlledBranch", "Branch") + .WithMany() + .HasForeignKey("BranchId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_branches_branch_id"); + + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany() + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Branch"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => + { + b.HasOne("marlin.web.Models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_update_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.HasOne("marlin.web.Models.Job", "Job") + .WithMany("Logs") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_logs_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_secrets_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany("Blobs") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany("Branches") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_commits_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithOne("WorkflowSnapshot") + .HasForeignKey("marlin.web.Models.WorkflowSnapshot", "CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_workflow_snapshots_source_commits_commit_id"); + + b.Navigation("Commit"); + }); + + modelBuilder.Entity("marlin.web.Models.Agent", b => + { + b.Navigation("Jobs"); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.Navigation("Logs"); + + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.Navigation("Blobs"); + + b.Navigation("Branches"); + + b.Navigation("WorkflowSnapshot"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/marlin.web/Migrations/20261027095603_AddLogEvent.cs b/src/marlin.web/Migrations/20261027095603_AddLogEvent.cs new file mode 100644 index 0000000..b32f68e --- /dev/null +++ b/src/marlin.web/Migrations/20261027095603_AddLogEvent.cs @@ -0,0 +1,67 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace marlin.web.Migrations +{ + /// <inheritdoc /> + public partial class AddLogEvent : Migration + { + /// <inheritdoc /> + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn<string>( + name: "main_branch", + table: "sources", + type: "text", + nullable: true, + oldClrType: typeof(string), + oldType: "text"); + + migrationBuilder.CreateTable( + name: "logs", + columns: table => new + { + id = table.Column<Guid>(type: "uuid", nullable: false), + job_id = table.Column<Guid>(type: "uuid", nullable: false), + at = table.Column<DateTime>(type: "timestamp with time zone", nullable: false), + message = table.Column<string>(type: "text", nullable: false), + error = table.Column<bool>(type: "boolean", nullable: false), + step_id = table.Column<string>(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_logs", x => x.id); + table.ForeignKey( + name: "fk_logs_jobs_job_id", + column: x => x.job_id, + principalTable: "jobs", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_logs_job_id", + table: "logs", + column: "job_id"); + } + + /// <inheritdoc /> + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "logs"); + + migrationBuilder.AlterColumn<string>( + name: "main_branch", + table: "sources", + type: "text", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "text", + oldNullable: true); + } + } +} diff --git a/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.Designer.cs b/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.Designer.cs new file mode 100644 index 0000000..5a199ba --- /dev/null +++ b/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.Designer.cs @@ -0,0 +1,565 @@ +// <auto-generated /> +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using marlin.web.Data; + +#nullable disable + +namespace marlin.web.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20261027130151_JobStatusUpdatesTable")] + partial class JobStatusUpdatesTable + { + /// <inheritdoc /> + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("marlin.web.Models.Agent", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<string>("AuthToken") + .IsRequired() + .HasColumnType("text") + .HasColumnName("auth_token"); + + b.Property<string>("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property<DateTime>("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property<int>("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property<string>("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<Guid?>("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property<Guid>("BranchId") + .HasColumnType("uuid") + .HasColumnName("branch_id"); + + b.Property<string>("ClaimToken") + .HasColumnType("text") + .HasColumnName("claim_token"); + + b.Property<Guid>("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property<DateTime>("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property<string>("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property<string>("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property<Guid>("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property<int>("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("BranchId") + .HasDatabaseName("ix_jobs_branch_id"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_jobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<DateTime>("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property<Guid>("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property<int>("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_updates"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_updates_job_id"); + + b.ToTable("job_status_updates", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<DateTime>("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property<bool>("Error") + .HasColumnType("boolean") + .HasColumnName("error"); + + b.Property<Guid>("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property<string>("Message") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message"); + + b.Property<string>("StepId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("step_id"); + + b.HasKey("Id") + .HasName("pk_logs"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_logs_job_id"); + + b.ToTable("logs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<string>("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property<string>("Kind") + .HasColumnType("text") + .HasColumnName("kind"); + + b.Property<string>("Label") + .IsRequired() + .HasColumnType("text") + .HasColumnName("label"); + + b.Property<bool>("MayReveal") + .HasColumnType("boolean") + .HasColumnName("may_reveal"); + + b.Property<Guid>("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_secrets"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_secrets_source_id"); + + b.ToTable("secrets", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<Guid?>("AccessSecretId") + .HasColumnType("uuid") + .HasColumnName("access_secret_id"); + + b.Property<string>("BranchFilter") + .IsRequired() + .HasColumnType("text") + .HasColumnName("branch_filter"); + + b.Property<string>("MainBranch") + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property<int>("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + + b.Property<string>("UpstreamUrl") + .IsRequired() + .HasColumnType("text") + .HasColumnName("upstream_url"); + + b.HasKey("Id") + .HasName("pk_sources"); + + b.ToTable("sources", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<Guid>("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property<byte[]>("Content") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.Property<string>("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property<Guid>("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_blobs"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_blobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_blobs_source_id"); + + b.ToTable("source_blobs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<Guid>("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property<string>("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property<Guid>("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_branches_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<bool>("BlobsFetched") + .HasColumnType("boolean") + .HasColumnName("blobs_fetched"); + + b.Property<string>("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property<Guid>("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property<bool>("WorkflowScanned") + .HasColumnType("boolean") + .HasColumnName("workflow_scanned"); + + b.HasKey("Id") + .HasName("pk_source_commits"); + + b.HasAlternateKey("SourceId", "Hash") + .HasName("ak_source_commits_source_id_hash"); + + b.ToTable("source_commits", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<Guid>("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property<DateTime>("ScannedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("scanned_at"); + + b.Property<string>("Workflow") + .IsRequired() + .HasColumnType("text") + .HasColumnName("workflow"); + + b.HasKey("Id") + .HasName("pk_workflow_snapshots"); + + b.HasIndex("CommitId") + .IsUnique() + .HasDatabaseName("ix_workflow_snapshots_commit_id"); + + b.ToTable("workflow_snapshots", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.HasOne("marlin.web.Models.Agent", "Agent") + .WithMany("Jobs") + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.web.Models.VersionControlledBranch", "Branch") + .WithMany() + .HasForeignKey("BranchId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_branches_branch_id"); + + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany() + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Branch"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => + { + b.HasOne("marlin.web.Models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_updates_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.HasOne("marlin.web.Models.Job", "Job") + .WithMany("Logs") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_logs_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_secrets_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany("Blobs") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany("Branches") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_commits_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithOne("WorkflowSnapshot") + .HasForeignKey("marlin.web.Models.WorkflowSnapshot", "CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_workflow_snapshots_source_commits_commit_id"); + + b.Navigation("Commit"); + }); + + modelBuilder.Entity("marlin.web.Models.Agent", b => + { + b.Navigation("Jobs"); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.Navigation("Logs"); + + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.Navigation("Blobs"); + + b.Navigation("Branches"); + + b.Navigation("WorkflowSnapshot"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.cs b/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.cs new file mode 100644 index 0000000..9cf6546 --- /dev/null +++ b/src/marlin.web/Migrations/20261027130151_JobStatusUpdatesTable.cs @@ -0,0 +1,78 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace marlin.web.Migrations +{ + /// <inheritdoc /> + public partial class JobStatusUpdatesTable : Migration + { + /// <inheritdoc /> + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "fk_job_status_update_jobs_job_id", + table: "job_status_update"); + + migrationBuilder.DropPrimaryKey( + name: "pk_job_status_update", + table: "job_status_update"); + + migrationBuilder.RenameTable( + name: "job_status_update", + newName: "job_status_updates"); + + migrationBuilder.RenameIndex( + name: "ix_job_status_update_job_id", + table: "job_status_updates", + newName: "ix_job_status_updates_job_id"); + + migrationBuilder.AddPrimaryKey( + name: "pk_job_status_updates", + table: "job_status_updates", + column: "id"); + + migrationBuilder.AddForeignKey( + name: "fk_job_status_updates_jobs_job_id", + table: "job_status_updates", + column: "job_id", + principalTable: "jobs", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + } + + /// <inheritdoc /> + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "fk_job_status_updates_jobs_job_id", + table: "job_status_updates"); + + migrationBuilder.DropPrimaryKey( + name: "pk_job_status_updates", + table: "job_status_updates"); + + migrationBuilder.RenameTable( + name: "job_status_updates", + newName: "job_status_update"); + + migrationBuilder.RenameIndex( + name: "ix_job_status_updates_job_id", + table: "job_status_update", + newName: "ix_job_status_update_job_id"); + + migrationBuilder.AddPrimaryKey( + name: "pk_job_status_update", + table: "job_status_update", + column: "id"); + + migrationBuilder.AddForeignKey( + name: "fk_job_status_update_jobs_job_id", + table: "job_status_update", + column: "job_id", + principalTable: "jobs", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + } + } +} diff --git a/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs b/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs index 411706b..da08d88 100644 --- a/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs @@ -141,12 +141,50 @@ namespace marlin.web.Migrations .HasColumnName("status"); b.HasKey("Id") - .HasName("pk_job_status_update"); + .HasName("pk_job_status_updates"); b.HasIndex("JobId") - .HasDatabaseName("ix_job_status_update_job_id"); + .HasDatabaseName("ix_job_status_updates_job_id"); - b.ToTable("job_status_update", (string)null); + b.ToTable("job_status_updates", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property<DateTime>("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property<bool>("Error") + .HasColumnType("boolean") + .HasColumnName("error"); + + b.Property<Guid>("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property<string>("Message") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message"); + + b.Property<string>("StepId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("step_id"); + + b.HasKey("Id") + .HasName("pk_logs"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_logs_job_id"); + + b.ToTable("logs", (string)null); }); modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => @@ -204,7 +242,6 @@ namespace marlin.web.Migrations .HasColumnName("branch_filter"); b.Property<string>("MainBranch") - .IsRequired() .HasColumnType("text") .HasColumnName("main_branch"); @@ -399,7 +436,19 @@ namespace marlin.web.Migrations .HasForeignKey("JobId") .OnDelete(DeleteBehavior.Cascade) .IsRequired() - .HasConstraintName("fk_job_status_update_jobs_job_id"); + .HasConstraintName("fk_job_status_updates_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.web.Models.LogEventRecord", b => + { + b.HasOne("marlin.web.Models.Job", "Job") + .WithMany("Logs") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_logs_jobs_job_id"); b.Navigation("Job"); }); @@ -489,6 +538,8 @@ namespace marlin.web.Migrations modelBuilder.Entity("marlin.web.Models.Job", b => { + b.Navigation("Logs"); + b.Navigation("StatusUpdates"); }); diff --git a/src/marlin.web/Models/Job.cs b/src/marlin.web/Models/Job.cs index bd60773..f7b6278 100644 --- a/src/marlin.web/Models/Job.cs +++ b/src/marlin.web/Models/Job.cs @@ -1,5 +1,7 @@ namespace marlin.web.Models; +using core.models; + public class Job { public Guid Id { get; set; } @@ -25,6 +27,8 @@ public class Job public string Recipe { get; set; } // plan + + public ICollection<LogEventRecord> Logs { get; set; } } public class JobStatusUpdate @@ -36,13 +40,4 @@ public class JobStatusUpdate public DateTime At { get; set; } public JobStatus Status { get; set; } -} - -public enum JobStatus -{ - Pending, - Active, - Cancelled, - Success, - Failed }
\ No newline at end of file diff --git a/src/marlin.web/Models/LogEventRecord.cs b/src/marlin.web/Models/LogEventRecord.cs new file mode 100644 index 0000000..1a916d8 --- /dev/null +++ b/src/marlin.web/Models/LogEventRecord.cs @@ -0,0 +1,16 @@ +namespace marlin.web.Models; + +using core.models; + +public class LogEventRecord +{ + public Guid Id { get; set; } + + public Guid JobId { get; set; } + public Job? Job { get; set; } + + public DateTime At { get; set; } + public string Message { get; set; } + public bool Error { get; set; } + public string StepId { get; set; } +}
\ No newline at end of file diff --git a/src/marlin.web/Program.cs b/src/marlin.web/Program.cs index 700e696..8dea3fe 100644 --- a/src/marlin.web/Program.cs +++ b/src/marlin.web/Program.cs @@ -1,12 +1,16 @@ +using marlin.web.Controllers.Api; using marlin.web.Data; using marlin.web.Models; using marlin.web.Services; using Microsoft.EntityFrameworkCore; +using ProtoBuf.Grpc.Server; DotNetEnv.Env.NoClobber().TraversePath().Load(); var builder = WebApplication.CreateBuilder(args); +builder.Services.AddCodeFirstGrpc(); + builder.Services.AddDbContext<ApplicationDbContext>(options => options .UseNpgsql(builder.Configuration.GetConnectionString("Master")) .UseSnakeCaseNamingConvention()); @@ -20,7 +24,7 @@ builder.Services builder.Services.AddAuthorization(options => { options.AddPolicy("Agent", p => p.RequireClaim("agent_id")); - options.AddPolicy("JobScoped", p => p.RequireClaim("job_id")); + options.AddPolicy("JobScoped", p => p.RequireClaim("agent_id").RequireClaim("job_id")); }); builder.Services.AddHostedService<DatabaseUpgrader>(); @@ -49,6 +53,8 @@ app.UseRouting(); app.UseAuthentication(); app.UseAuthorization(); +app.MapGrpcService<InstanceGrpcService>(); + app.MapStaticAssets(); app diff --git a/src/marlin.web/Services/VersionControlSyncService.cs b/src/marlin.web/Services/VersionControlSyncService.cs index 8f2046e..4509745 100644 --- a/src/marlin.web/Services/VersionControlSyncService.cs +++ b/src/marlin.web/Services/VersionControlSyncService.cs @@ -115,7 +115,7 @@ public class VersionControlSyncService : BackgroundService private async Task<string> GetLocalPath(IGitClient client, VersionControlSource source) { - var dir = Path.Join(Environment.CurrentDirectory, "run", "sources", source.Id.ToString("N")); + var dir = Path.Join(CommonPaths.WebRoot, "sources", source.Id.ToString("N")); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); diff --git a/src/marlin.web/Views/Jobs/Details.cshtml b/src/marlin.web/Views/Jobs/Details.cshtml index ebba94d..9b27fe3 100644 --- a/src/marlin.web/Views/Jobs/Details.cshtml +++ b/src/marlin.web/Views/Jobs/Details.cshtml @@ -2,4 +2,27 @@ @Model.Id -<a asp-action="List">Back</a>
\ No newline at end of file +@Model.Branch.Name + +<dl> + <dt>Id</dt> + <dd>@Model.Id</dd> + + <dt>Branch</dt> + <dd>@Model.Branch.Name</dd> + + <dt>Commit</dt> + <dd>@Model.Commit.Hash</dd> + + <dt>Status</dt> + <dd>@Model.Status</dd> +</dl> + +<ul> + @foreach (var line in Model.Logs.OrderBy(x=>x.At)) + { + <li style="color: @(line.Error ? "red" : "black")">[@line.At.ToString("HH:mm:ss")] [@line.StepId] @line.Message</li> + } +</ul> + +<a asp-action="List">Back</a> diff --git a/src/marlin.web/marlin.web.csproj b/src/marlin.web/marlin.web.csproj index 26c8ea3..b931039 100644 --- a/src/marlin.web/marlin.web.csproj +++ b/src/marlin.web/marlin.web.csproj @@ -15,6 +15,8 @@ <PrivateAssets>all</PrivateAssets> </PackageReference> <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.2" /> + <PackageReference Include="protobuf-net.Grpc" Version="1.3.14" /> + <PackageReference Include="protobuf-net.Grpc.AspNetCore" Version="1.3.14" /> </ItemGroup> <ItemGroup> |