From a9bc03fb01b5d1ac41bf5592f384d22df5425aa4 Mon Sep 17 00:00:00 2001 From: Melodi <62381002+Melodi17@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:02:31 +1000 Subject: feat: vcs and secret management support --- src/marlin.core/functional/GitBinaryClient.cs | 133 +++++++ src/marlin.core/functional/GitCommandException.cs | 16 + src/marlin.core/functional/IGitClient.cs | 69 +--- src/marlin.core/functional/IGitCredential.cs | 18 + src/marlin.core/functional/NoopAsyncDisposable.cs | 10 + src/marlin.core/functional/SshKeyGitCredential.cs | 57 +++ src/marlin.core/functional/TokenGitCredential.cs | 32 ++ src/marlin.core/models/ISecret.cs | 11 + src/marlin.core/models/ISourceSecretStore.cs | 9 + src/marlin.core/models/Job.cs | 2 +- src/marlin.core/models/SecretKind.cs | 7 + src/marlin.core/models/VersionControlSource.cs | 13 +- src/marlin.core/models/VersionControlledBlob.cs | 15 + src/marlin.core/models/VersionControlledBranch.cs | 14 + src/marlin.core/models/VersionControlledCommit.cs | 15 + src/marlin.core/util/ISecretExtensions.cs | 12 + src/marlin.web/Controllers/JobsController.cs | 31 +- src/marlin.web/Controllers/VCSController.cs | 92 ++++- src/marlin.web/Data/ApplicationDbContext.cs | 32 +- .../20260902214847_InitialCreate.Designer.cs | 53 --- .../Migrations/20260902214847_InitialCreate.cs | 35 -- .../20260903141622_InitialCreate.Designer.cs | 259 +++++++++++++ .../Migrations/20260903141622_InitialCreate.cs | 154 ++++++++ .../20260905011100_AddCommitModel.Designer.cs | 376 ++++++++++++++++++ .../Migrations/20260905011100_AddCommitModel.cs | 126 ++++++ .../20260905102259_AddSecretModel.Designer.cs | 430 +++++++++++++++++++++ .../Migrations/20260905102259_AddSecretModel.cs | 59 +++ .../20260905104308_BlobNowBinary.Designer.cs | 430 +++++++++++++++++++++ .../Migrations/20260905104308_BlobNowBinary.cs | 40 ++ .../ApplicationDbContextModelSnapshot.cs | 379 +++++++++++++++++- src/marlin.web/Models/DatabaseSourceSecretStore.cs | 27 ++ src/marlin.web/Program.cs | 8 +- .../Services/VersionControlSyncService.cs | 201 +++++++++- src/marlin.web/Views/Jobs/Details.cshtml | 5 + src/marlin.web/Views/Jobs/List.cshtml | 4 + src/marlin.web/Views/Jobs/Spawn.cshtml | 56 +++ src/marlin.web/Views/Shared/_Layout.cshtml | 5 +- src/marlin.web/Views/VCS/Create.cshtml | 8 +- src/marlin.web/Views/VCS/CreateSecret.cshtml | 12 + src/marlin.web/Views/VCS/Details.cshtml | 44 ++- src/marlin.web/Views/VCS/List.cshtml | 2 +- src/marlin.web/Views/VCS/ListSecret.cshtml | 45 +++ src/marlin.web/appsettings.json | 3 +- src/marlin.web/marlin.web.csproj | 4 + src/marlin.web/wwwroot/css/site.css | 20 + 45 files changed, 3175 insertions(+), 198 deletions(-) create mode 100644 src/marlin.core/functional/GitBinaryClient.cs create mode 100644 src/marlin.core/functional/GitCommandException.cs create mode 100644 src/marlin.core/functional/IGitCredential.cs create mode 100644 src/marlin.core/functional/NoopAsyncDisposable.cs create mode 100644 src/marlin.core/functional/SshKeyGitCredential.cs create mode 100644 src/marlin.core/functional/TokenGitCredential.cs create mode 100644 src/marlin.core/models/ISecret.cs create mode 100644 src/marlin.core/models/ISourceSecretStore.cs create mode 100644 src/marlin.core/models/SecretKind.cs create mode 100644 src/marlin.core/models/VersionControlledBlob.cs create mode 100644 src/marlin.core/models/VersionControlledBranch.cs create mode 100644 src/marlin.core/models/VersionControlledCommit.cs create mode 100644 src/marlin.core/util/ISecretExtensions.cs delete mode 100644 src/marlin.web/Migrations/20260902214847_InitialCreate.Designer.cs delete mode 100644 src/marlin.web/Migrations/20260902214847_InitialCreate.cs create mode 100644 src/marlin.web/Migrations/20260903141622_InitialCreate.Designer.cs create mode 100644 src/marlin.web/Migrations/20260903141622_InitialCreate.cs create mode 100644 src/marlin.web/Migrations/20260905011100_AddCommitModel.Designer.cs create mode 100644 src/marlin.web/Migrations/20260905011100_AddCommitModel.cs create mode 100644 src/marlin.web/Migrations/20260905102259_AddSecretModel.Designer.cs create mode 100644 src/marlin.web/Migrations/20260905102259_AddSecretModel.cs create mode 100644 src/marlin.web/Migrations/20260905104308_BlobNowBinary.Designer.cs create mode 100644 src/marlin.web/Migrations/20260905104308_BlobNowBinary.cs create mode 100644 src/marlin.web/Models/DatabaseSourceSecretStore.cs create mode 100644 src/marlin.web/Views/Jobs/Details.cshtml create mode 100644 src/marlin.web/Views/Jobs/Spawn.cshtml create mode 100644 src/marlin.web/Views/VCS/CreateSecret.cshtml create mode 100644 src/marlin.web/Views/VCS/ListSecret.cshtml (limited to 'src') diff --git a/src/marlin.core/functional/GitBinaryClient.cs b/src/marlin.core/functional/GitBinaryClient.cs new file mode 100644 index 0000000..001f2c8 --- /dev/null +++ b/src/marlin.core/functional/GitBinaryClient.cs @@ -0,0 +1,133 @@ +namespace marlin.core.functional; + +using System.Diagnostics; + +public class GitBinaryClient : IGitClient +{ + private readonly IGitCredential? _credential; + + public GitBinaryClient(IGitCredential? credential = null) + { + this._credential = credential; + } + + public async Task> GetRemoteRefs(string remote) + { + string stdout = await this.RunAsync( + authenticate: true, + "ls-remote", "--heads", "--tags", remote); + + return ParseRefs(stdout); + } + + public async Task Init(string repoPath) + { + Directory.CreateDirectory(repoPath); + await this.RunAsync(authenticate: false, "init", "--bare", repoPath); + } + + public async Task FetchTrees(string repoPath, string remote, IEnumerable refs, bool skipBlobs = false) + { + var refList = refs as IReadOnlyCollection ?? refs.ToArray(); + if (refList.Count == 0) + return; + + var args = new List { "-C", repoPath, "fetch" }; + if (skipBlobs) + args.Add("--filter=blob:none"); + args.Add(remote); + args.AddRange(refList); + + await this.RunAsync(authenticate: true, args.ToArray()); + } + + public async Task ListTree(string repoPath, string hash, string filterPath = ".") + { + string stdout = await this.RunAsync( + authenticate: false, + "-C", repoPath, "ls-tree", "-r", "--name-only", hash, "--", filterPath); + + return stdout.Split('\n', StringSplitOptions.RemoveEmptyEntries); + } + + public Task CatFile(string repoPath, string hash, string fileName) + { + return this.RunRawAsync( + authenticate: false, + "-C", repoPath, "show", $"{hash}:{fileName}"); + } + + private Task RunAsync(bool authenticate, params string[] arguments) => + this.RunAsync(authenticate, async stream => + { + using var reader = new StreamReader(stream); + return await reader.ReadToEndAsync(); + }, arguments); + + private Task RunRawAsync(bool authenticate, params string[] arguments) => + this.RunAsync(authenticate, async stream => + { + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer); + return buffer.ToArray(); + }, arguments); + + private async Task RunAsync(bool authenticate, Func> readStdout, string[] arguments) + { + var psi = new ProcessStartInfo("git") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + foreach (string arg in arguments) + psi.ArgumentList.Add(arg); + + psi.Environment["GIT_TERMINAL_PROMPT"] = "0"; + + IAsyncDisposable? credentialScope = authenticate && this._credential != null + ? await this._credential.ApplyAsync(psi) + : null; + + try + { + using var process = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start git process."); + + Task stdoutTask = readStdout(process.StandardOutput.BaseStream); + Task stderrTask = process.StandardError.ReadToEndAsync(); + + await process.WaitForExitAsync(); + T stdout = await stdoutTask; + string stderr = await stderrTask; + + if (process.ExitCode != 0) + throw new GitCommandException(string.Join(' ', arguments), process.ExitCode, stderr); + + return stdout; + } + finally + { + if (credentialScope != null) + await credentialScope.DisposeAsync(); + } + } + + private static Dictionary ParseRefs(string stdout) + { + var result = new Dictionary(); + + foreach (string line in stdout.Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + string[] parts = line.Split('\t'); + if (parts.Length != 2) + continue; // ignore a malformed line rather than throw over one bad entry + + result[parts[1]] = parts[0]; + } + + return result; + } +} diff --git a/src/marlin.core/functional/GitCommandException.cs b/src/marlin.core/functional/GitCommandException.cs new file mode 100644 index 0000000..e2ee42d --- /dev/null +++ b/src/marlin.core/functional/GitCommandException.cs @@ -0,0 +1,16 @@ +namespace marlin.core.functional; + +public sealed class GitCommandException : Exception +{ + public string Command { get; } + public int ExitCode { get; } + public string StandardError { get; } + + public GitCommandException(string command, int exitCode, string standardError) + : base($"git {command} exited with code {exitCode}: {standardError.Trim()}") + { + this.Command = command; + this.ExitCode = exitCode; + this.StandardError = standardError; + } +} diff --git a/src/marlin.core/functional/IGitClient.cs b/src/marlin.core/functional/IGitClient.cs index 04dc12a..69cffa3 100644 --- a/src/marlin.core/functional/IGitClient.cs +++ b/src/marlin.core/functional/IGitClient.cs @@ -1,71 +1,10 @@ namespace marlin.core.functional; -using System.Diagnostics; - public interface IGitClient { Task> GetRemoteRefs(string remote); -} - -public class GitBinaryClient : IGitClient -{ - public virtual async Task> GetRemoteRefs(string remote) - { - var psi = this.CreateProcess(remote); - var proc = Process.Start(psi)!; - - string stdout = await proc.StandardOutput.ReadToEndAsync(); - await proc.WaitForExitAsync(); - - return ParseOutput(stdout); - } - - private Dictionary ParseOutput(string stdout) - { - return stdout - .Split("\n", StringSplitOptions.RemoveEmptyEntries) - .Select(x => x.Split('\t')) - .ToDictionary(k => k[1], k => k[0]); - } - - protected virtual ProcessStartInfo CreateProcess(string remote) - { - var psi = new ProcessStartInfo("git", $"ls-remote --heads --tags {remote}") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - Environment = { ["GIT_TERMINAL_PROMPT"] = "0" } - }; - - return psi; - } - - /*private async Task SshAuth(string privateKeyPem) - { - string keyPath = Path.Combine(Path.GetTempPath(), $"key_{Guid.NewGuid():N}"); - await File.WriteAllTextAsync(keyPath, privateKeyPem); - File.SetUnixFileMode(keyPath, UnixFileMode.UserRead | UnixFileMode.UserWrite); - - try - { - psi.Environment["GIT_SSH_COMMAND"] = - $"ssh -i {keyPath} -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new"; - // run ls-remote against the ssh:// or git@ URL - } - finally - { - File.Delete(keyPath); - } - } - - private async Task BasicAuth(string accessToken) - { - psi.Environment["GIT_CONFIG_COUNT"] = "1"; - psi.Environment["GIT_CONFIG_KEY_0"] = "http.extraheader"; - psi.Environment["GIT_CONFIG_VALUE_0"] = - $"AUTHORIZATION: basic {Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"))}"; - - }*/ + Task Init(string repoPath); + Task FetchTrees(string repoPath, string remote, IEnumerable refs, bool skipBlobs = false); + Task ListTree(string repoPath, string hash, string filterPath = "."); + Task CatFile(string repoPath, string hash, string fileName); } \ No newline at end of file diff --git a/src/marlin.core/functional/IGitCredential.cs b/src/marlin.core/functional/IGitCredential.cs new file mode 100644 index 0000000..cf96804 --- /dev/null +++ b/src/marlin.core/functional/IGitCredential.cs @@ -0,0 +1,18 @@ +namespace marlin.core.functional; + +using System.Diagnostics; + +/// +/// Supplies whatever git needs to authenticate a single invocation — an HTTP header, +/// an SSH key, etc. Credentials are applied per-process via environment variables only; +/// nothing is written to git config, argv, or any credential store. +/// +public interface IGitCredential +{ + /// + /// Configures so the git process it starts can authenticate. + /// Returns a disposable that tears down anything ephemeral (e.g. a temp key file) + /// once that process has exited. + /// + Task ApplyAsync(ProcessStartInfo psi); +} diff --git a/src/marlin.core/functional/NoopAsyncDisposable.cs b/src/marlin.core/functional/NoopAsyncDisposable.cs new file mode 100644 index 0000000..330a197 --- /dev/null +++ b/src/marlin.core/functional/NoopAsyncDisposable.cs @@ -0,0 +1,10 @@ +namespace marlin.core.functional; + +internal sealed class NoopAsyncDisposable : IAsyncDisposable +{ + public static readonly NoopAsyncDisposable Instance = new(); + + private NoopAsyncDisposable() { } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} diff --git a/src/marlin.core/functional/SshKeyGitCredential.cs b/src/marlin.core/functional/SshKeyGitCredential.cs new file mode 100644 index 0000000..9183a66 --- /dev/null +++ b/src/marlin.core/functional/SshKeyGitCredential.cs @@ -0,0 +1,57 @@ +namespace marlin.core.functional; + +using System.Diagnostics; + +/// +/// SSH auth via a private key. The key is written to a per-invocation temp file +/// (owner-read/write only) and deleted as soon as the git process that used it exits — +/// ssh has no clean way to take key material via stdin/env, so a short-lived file is +/// the least-bad option. +/// +public sealed class SshKeyGitCredential : IGitCredential +{ + private readonly string _privateKeyPem; + + public SshKeyGitCredential(string privateKeyPem) + { + this._privateKeyPem = privateKeyPem; + } + + public async Task ApplyAsync(ProcessStartInfo psi) + { + string keyPath = Path.Combine(Path.GetTempPath(), $"marlin_{Guid.NewGuid():N}.key"); + await File.WriteAllTextAsync(keyPath, this._privateKeyPem); + + if (!OperatingSystem.IsWindows()) + File.SetUnixFileMode(keyPath, UnixFileMode.UserRead | UnixFileMode.UserWrite); + + psi.Environment["GIT_SSH_COMMAND"] = + $"ssh -i \"{keyPath}\" -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new"; + + return new TempKeyFile(keyPath); + } + + private sealed class TempKeyFile : IAsyncDisposable + { + private readonly string _path; + + public TempKeyFile(string path) + { + this._path = path; + } + + public ValueTask DisposeAsync() + { + try + { + File.Delete(this._path); + } + catch (IOException) + { + // best-effort cleanup; a leftover temp file is not worth failing the caller over + } + + return ValueTask.CompletedTask; + } + } +} diff --git a/src/marlin.core/functional/TokenGitCredential.cs b/src/marlin.core/functional/TokenGitCredential.cs new file mode 100644 index 0000000..5dfef66 --- /dev/null +++ b/src/marlin.core/functional/TokenGitCredential.cs @@ -0,0 +1,32 @@ +namespace marlin.core.functional; + +using System.Diagnostics; +using System.Text; + +/// +/// HTTPS auth via a bearer/basic token. Injected as an Authorization header for a +/// single git invocation through GIT_CONFIG_* env vars, so it never appears in argv +/// (visible via ps) or in git's own error output (which echoes the URL, not headers). +/// +public sealed class TokenGitCredential : IGitCredential +{ + private readonly string _username; + private readonly string _token; + + public TokenGitCredential(string token, string username = "x-access-token") + { + this._token = token; + this._username = username; + } + + public Task ApplyAsync(ProcessStartInfo psi) + { + string header = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{this._username}:{this._token}")); + + psi.Environment["GIT_CONFIG_COUNT"] = "1"; + psi.Environment["GIT_CONFIG_KEY_0"] = "http.extraheader"; + psi.Environment["GIT_CONFIG_VALUE_0"] = $"AUTHORIZATION: basic {header}"; + + return Task.FromResult(NoopAsyncDisposable.Instance); + } +} diff --git a/src/marlin.core/models/ISecret.cs b/src/marlin.core/models/ISecret.cs new file mode 100644 index 0000000..e41a3b9 --- /dev/null +++ b/src/marlin.core/models/ISecret.cs @@ -0,0 +1,11 @@ +namespace marlin.web.Models; + +public interface ISecret +{ + Guid Id { get; set; } + string Label { get; set; } + string? Kind { get; set; } + bool MayReveal { get; set; } + + Dictionary Content { get; set; } +} \ No newline at end of file diff --git a/src/marlin.core/models/ISourceSecretStore.cs b/src/marlin.core/models/ISourceSecretStore.cs new file mode 100644 index 0000000..0d5053e --- /dev/null +++ b/src/marlin.core/models/ISourceSecretStore.cs @@ -0,0 +1,9 @@ +namespace marlin.web.Models; + +using core.functional; +using core.models; + +public interface ISourceSecretStore +{ + Task GetAsync(Guid id, CancellationToken ct); +} \ No newline at end of file diff --git a/src/marlin.core/models/Job.cs b/src/marlin.core/models/Job.cs index 6b09597..a8c7c5d 100644 --- a/src/marlin.core/models/Job.cs +++ b/src/marlin.core/models/Job.cs @@ -14,7 +14,7 @@ public class Job public JobStatus Status { get; set; } public ICollection StatusUpdates { get; set; } - public Guid AgentId { get; set; } + public Guid? AgentId { get; set; } public Agent? Agent { get; set; } public string Recipe { get; set; } diff --git a/src/marlin.core/models/SecretKind.cs b/src/marlin.core/models/SecretKind.cs new file mode 100644 index 0000000..02652d5 --- /dev/null +++ b/src/marlin.core/models/SecretKind.cs @@ -0,0 +1,7 @@ +namespace marlin.web.Models; + +public static class SecretKind +{ + public const string HTTPToken = "http-token"; + public const string SSHKey = "ssh-key"; +} \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlSource.cs b/src/marlin.core/models/VersionControlSource.cs index 1d50ef3..daa8d51 100644 --- a/src/marlin.core/models/VersionControlSource.cs +++ b/src/marlin.core/models/VersionControlSource.cs @@ -9,16 +9,7 @@ public class VersionControlSource public string BranchFilter { get; set; } public int MinPollingInterval { get; set; } - public ICollection Branches { get; set; } -} - -public class VersionControlledBranch -{ - public Guid Id { get; set; } - - public Guid SourceId { get; set; } - public VersionControlSource Source { get; set; } + public Guid? AccessSecretId { get; set; } - public string Name { get; set; } - public string HeadCommit { get; set; } + public ICollection Branches { get; set; } } \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlledBlob.cs b/src/marlin.core/models/VersionControlledBlob.cs new file mode 100644 index 0000000..03915be --- /dev/null +++ b/src/marlin.core/models/VersionControlledBlob.cs @@ -0,0 +1,15 @@ +namespace marlin.core.models; + +public class VersionControlledBlob +{ + public Guid Id { get; set; } + + public Guid SourceId { get; set; } + public VersionControlSource Source { get; set; } + + public Guid CommitId { get; set; } + public VersionControlledCommit Commit { get; set; } + + public string Path { get; set; } + public byte[] Content { get; set; } +} \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlledBranch.cs b/src/marlin.core/models/VersionControlledBranch.cs new file mode 100644 index 0000000..601f527 --- /dev/null +++ b/src/marlin.core/models/VersionControlledBranch.cs @@ -0,0 +1,14 @@ +namespace marlin.core.models; + +public class VersionControlledBranch +{ + public Guid Id { get; set; } + + public Guid SourceId { get; set; } + public VersionControlSource Source { get; set; } + + public Guid CommitId { get; set; } + public VersionControlledCommit Commit { get; set; } + + public string Name { get; set; } +} \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlledCommit.cs b/src/marlin.core/models/VersionControlledCommit.cs new file mode 100644 index 0000000..a7b5e9e --- /dev/null +++ b/src/marlin.core/models/VersionControlledCommit.cs @@ -0,0 +1,15 @@ +namespace marlin.core.models; + +public class VersionControlledCommit +{ + public Guid Id { get; set; } + + public Guid SourceId { get; set; } + public VersionControlSource Source { get; set; } + public bool BlobsFetched { get; set; } + + public string Hash { get; set; } + + public ICollection Branches { get; set; } + public ICollection Blobs { get; set; } +} \ No newline at end of file diff --git a/src/marlin.core/util/ISecretExtensions.cs b/src/marlin.core/util/ISecretExtensions.cs new file mode 100644 index 0000000..e40c77c --- /dev/null +++ b/src/marlin.core/util/ISecretExtensions.cs @@ -0,0 +1,12 @@ +namespace marlin.core.util; + +using System.Runtime.CompilerServices; +using web.Models; + +public static class ISecretExtensions +{ + public static bool Has(this ISecret secret, params string[] keys) + { + return keys.All(x => secret.Content.ContainsKey(x)); + } +} \ No newline at end of file diff --git a/src/marlin.web/Controllers/JobsController.cs b/src/marlin.web/Controllers/JobsController.cs index 1cfdadb..c86cb2d 100644 --- a/src/marlin.web/Controllers/JobsController.cs +++ b/src/marlin.web/Controllers/JobsController.cs @@ -30,10 +30,23 @@ public class JobsController(ApplicationDbContext dbContext) : Controller return this.View(results); } + public record JobSpawnDetails( + ICollection Sources, + Guid? Vcs = null, + string? Ref = null, + string? Commit = null, + string? Recipe = null); + [HttpGet("spawn")] - public async Task Spawn() + public async Task Spawn(CancellationToken ct) { - return this.View(); + var sources = await dbContext.Sources + .Include(x => x.Branches) + .ThenInclude(x => x.Commit) + .ToListAsync(ct); + + var details = new JobSpawnDetails(sources); + return this.View(details); } public record JobSpawnRequest(Guid Vcs, string Ref, string Commit, string Recipe); @@ -62,4 +75,18 @@ public class JobsController(ApplicationDbContext dbContext) : Controller // cancel job // details job (recipe, status, agent, target, plan, log, artifacts) + [HttpGet("details/{id}")] + public async Task Details(Guid id) + { + var job = await dbContext.Jobs + .Include(x => x.Agent) + .Include(x => x.Source) + .Include(x => x.StatusUpdates) + .FirstOrDefaultAsync(x => x.Id == id); + + if (job == null) + return this.NotFound(); + + return this.View(job); + } } \ No newline at end of file diff --git a/src/marlin.web/Controllers/VCSController.cs b/src/marlin.web/Controllers/VCSController.cs index 86f655b..7370e0d 100644 --- a/src/marlin.web/Controllers/VCSController.cs +++ b/src/marlin.web/Controllers/VCSController.cs @@ -1,33 +1,47 @@ namespace marlin.web.Controllers; +using System.Text.Json; using core; using core.models; using Data; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using Models; +using Services; [Route("vcs")] -public class VCSController(ApplicationDbContext dbContext) : Controller +public class VCSController(ApplicationDbContext dbContext, VersionControlSyncService syncService) : Controller { [HttpGet("list")] public async Task List(CancellationToken ct) { var items = await dbContext.Sources + .Include(x => x.Branches) .ToListAsync(ct); return this.View(items); } + public record SecretOption(Guid Id, string Label); + [HttpGet("details/{id}")] public async Task Details(Guid id, CancellationToken ct) { var item = await dbContext.Sources + .Include(x => x.Branches) + .ThenInclude(x => x.Commit) .FirstOrDefaultAsync(x => x.Id == id, ct); if (item == null) return this.NotFound(); - return this.View(item); + // Only Id/Label projected — a secret's Content never needs to leave the secrets pages. + var secretOptions = await dbContext.Secrets + .Where(x => x.SourceId == id) + .Select(x => new SecretOption(x.Id, x.Label)) + .ToListAsync(ct); + + return this.View((item, secretOptions)); } [HttpPost("edit/{id}")] @@ -43,8 +57,10 @@ public class VCSController(ApplicationDbContext dbContext) : Controller item.BranchFilter = vcs.BranchFilter; item.MainBranch = vcs.MainBranch; item.MinPollingInterval = vcs.MinPollingInterval; + item.AccessSecretId = vcs.AccessSecretId; await dbContext.SaveChangesAsync(ct); + await syncService.FetchSources(ct); return this.RedirectToAction("List"); } @@ -62,7 +78,79 @@ public class VCSController(ApplicationDbContext dbContext) : Controller await dbContext.AddAsync(vcs, ct); await dbContext.SaveChangesAsync(ct); + await syncService.FetchSources(ct); return this.RedirectToAction("Details", new { id = vcs.Id }); } + + public record SecretCreateRequest(string Label, string? Kind, bool MayReveal, string ContentJson); + + [HttpGet("details/{sourceId}/secrets/create")] + public async Task CreateSecret(Guid sourceId, CancellationToken ct) + { + var source = await dbContext.Sources + .FirstOrDefaultAsync(x => x.Id == sourceId, ct); + + if (source == null) + return this.NotFound(); + + return this.View(source); + } + + [HttpPost("details/{sourceId}/secrets/create")] + public async Task CreateSecretPost(Guid sourceId, SecretCreateRequest request, CancellationToken ct) + { + var source = await dbContext.Sources + .FirstOrDefaultAsync(x => x.Id == sourceId, ct); + + if (source == null) + return this.NotFound(); + + var content = JsonSerializer.Deserialize>(request.ContentJson) ?? new(); + + var secret = new VersionControlSecret + { + SourceId = sourceId, + Label = request.Label, + Kind = request.Kind, + MayReveal = request.MayReveal, + Content = content + }; + + await dbContext.Secrets.AddAsync(secret, ct); + await dbContext.SaveChangesAsync(ct); + + return this.RedirectToAction("ListSecret", new { sourceId }); + } + + [HttpGet("details/{sourceId}/secrets/list")] + public async Task ListSecret(Guid sourceId, CancellationToken ct) + { + var source = await dbContext.Sources + .FirstOrDefaultAsync(x => x.Id == sourceId, ct); + + if (source == null) + return this.NotFound(); + + var items = await dbContext.Secrets + .Where(x => x.SourceId == source.Id) + .ToListAsync(ct); + + return this.View((source, items)); + } + + [HttpPost("details/{sourceId}/secrets/delete/{id}")] + public async Task DeleteSecret(Guid sourceId, Guid id, CancellationToken ct) + { + var secret = await dbContext.Secrets + .FirstOrDefaultAsync(x => x.SourceId == sourceId && x.Id == id, ct); + + if (secret == null) + return this.NotFound(); + + dbContext.Secrets.Remove(secret); + await dbContext.SaveChangesAsync(ct); + + return this.RedirectToAction("ListSecret", new { sourceId = sourceId }); + } } \ No newline at end of file diff --git a/src/marlin.web/Data/ApplicationDbContext.cs b/src/marlin.web/Data/ApplicationDbContext.cs index 41ec4bc..2e05835 100644 --- a/src/marlin.web/Data/ApplicationDbContext.cs +++ b/src/marlin.web/Data/ApplicationDbContext.cs @@ -1,15 +1,31 @@ namespace marlin.web.Data; using System.Linq.Expressions; +using System.Text.Json; using core; using core.models; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Models; + +// user-based session grant +// agent id/token grant +// agent job claim session grant +// service grant +public interface IActiveGrant +{ + bool CanAccessSource(Guid sourceId); +} public class ApplicationDbContext : DbContext { public DbSet Sources => this.Set(); + public DbSet SourceBranches => this.Set(); + public DbSet SourceCommits => this.Set(); + public DbSet SourceBlobs => this.Set(); + public DbSet Secrets => this.Set(); public DbSet Jobs => this.Set(); + public DbSet Agents => this.Set(); public ApplicationDbContext(DbContextOptions options) : base(options) { } @@ -18,7 +34,21 @@ public class ApplicationDbContext : DbContext { modelBuilder .Entity() - .OwnsOne(p => p.Capabilities, b => b.ToJson()); + .Property(p => p.Capabilities) + .HasConversion( + v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null), + v => JsonSerializer.Deserialize(v, (JsonSerializerOptions?)null)!); + + modelBuilder + .Entity() + .HasAlternateKey(x => new { x.SourceId, x.Hash }); + + modelBuilder + .Entity() + .Property(p => p.Content) + .HasConversion( + v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null), + v => JsonSerializer.Deserialize>(v, (JsonSerializerOptions?)null)!); base.OnModelCreating(modelBuilder); } diff --git a/src/marlin.web/Migrations/20260902214847_InitialCreate.Designer.cs b/src/marlin.web/Migrations/20260902214847_InitialCreate.Designer.cs deleted file mode 100644 index 21ab701..0000000 --- a/src/marlin.web/Migrations/20260902214847_InitialCreate.Designer.cs +++ /dev/null @@ -1,53 +0,0 @@ -// -using System; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; -using marlin.web.Data; - -#nullable disable - -namespace marlin.web.Migrations -{ - [DbContext(typeof(ApplicationDbContext))] - [Migration("20260902214847_InitialCreate")] - partial class InitialCreate - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "10.0.11") - .HasAnnotation("Relational:MaxIdentifierLength", 63); - - NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - - modelBuilder.Entity("marlin.core.VersionControlSource", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid") - .HasColumnName("id"); - - b.Property("BranchFilter") - .IsRequired() - .HasColumnType("text") - .HasColumnName("branch_filter"); - - b.Property("UpstreamUrl") - .IsRequired() - .HasColumnType("text") - .HasColumnName("upstream_url"); - - b.HasKey("Id") - .HasName("pk_sources"); - - b.ToTable("sources", (string)null); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/src/marlin.web/Migrations/20260902214847_InitialCreate.cs b/src/marlin.web/Migrations/20260902214847_InitialCreate.cs deleted file mode 100644 index 6c91890..0000000 --- a/src/marlin.web/Migrations/20260902214847_InitialCreate.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace marlin.web.Migrations -{ - /// - public partial class InitialCreate : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.CreateTable( - name: "sources", - columns: table => new - { - id = table.Column(type: "uuid", nullable: false), - upstream_url = table.Column(type: "text", nullable: false), - branch_filter = table.Column(type: "text", nullable: false) - }, - constraints: table => - { - table.PrimaryKey("pk_sources", x => x.id); - }); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropTable( - name: "sources"); - } - } -} diff --git a/src/marlin.web/Migrations/20260903141622_InitialCreate.Designer.cs b/src/marlin.web/Migrations/20260903141622_InitialCreate.Designer.cs new file mode 100644 index 0000000..ad71457 --- /dev/null +++ b/src/marlin.web/Migrations/20260903141622_InitialCreate.Designer.cs @@ -0,0 +1,259 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using marlin.web.Data; + +#nullable disable + +namespace marlin.web.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20260903141622_InitialCreate")] + partial class InitialCreate + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("marlin.core.models.Agent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property("Commit") + .IsRequired() + .HasColumnType("text") + .HasColumnName("commit"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property("Ref") + .IsRequired() + .HasColumnType("text") + .HasColumnName("ref"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_update"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_update_job_id"); + + b.ToTable("job_status_update", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BranchFilter") + .IsRequired() + .HasColumnType("text") + .HasColumnName("branch_filter"); + + b.Property("MainBranch") + .IsRequired() + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + + b.Property("UpstreamUrl") + .IsRequired() + .HasColumnType("text") + .HasColumnName("upstream_url"); + + b.HasKey("Id") + .HasName("pk_sources"); + + b.ToTable("sources", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("HeadCommit") + .IsRequired() + .HasColumnType("text") + .HasColumnName("head_commit"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.HasOne("marlin.core.models.Agent", "Agent") + .WithMany() + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.HasOne("marlin.core.models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_update_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/marlin.web/Migrations/20260903141622_InitialCreate.cs b/src/marlin.web/Migrations/20260903141622_InitialCreate.cs new file mode 100644 index 0000000..2ebcc67 --- /dev/null +++ b/src/marlin.web/Migrations/20260903141622_InitialCreate.cs @@ -0,0 +1,154 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace marlin.web.Migrations +{ + /// + public partial class InitialCreate : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "agents", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + name = table.Column(type: "text", nullable: false), + max_concurrent_jobs = table.Column(type: "integer", nullable: false), + last_seen = table.Column(type: "timestamp with time zone", nullable: false), + capabilities = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_agents", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "sources", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + upstream_url = table.Column(type: "text", nullable: false), + main_branch = table.Column(type: "text", nullable: false), + branch_filter = table.Column(type: "text", nullable: false), + min_polling_interval = table.Column(type: "integer", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_sources", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "jobs", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + source_id = table.Column(type: "uuid", nullable: false), + @ref = table.Column(name: "ref", type: "text", nullable: false), + commit = table.Column(type: "text", nullable: false), + created_reason = table.Column(type: "text", nullable: false), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + status = table.Column(type: "integer", nullable: false), + agent_id = table.Column(type: "uuid", nullable: true), + recipe = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_jobs", x => x.id); + table.ForeignKey( + name: "fk_jobs_agents_agent_id", + column: x => x.agent_id, + principalTable: "agents", + principalColumn: "id"); + table.ForeignKey( + name: "fk_jobs_sources_source_id", + column: x => x.source_id, + principalTable: "sources", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "source_branches", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + source_id = table.Column(type: "uuid", nullable: false), + name = table.Column(type: "text", nullable: false), + head_commit = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_source_branches", x => x.id); + table.ForeignKey( + name: "fk_source_branches_sources_source_id", + column: x => x.source_id, + principalTable: "sources", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "job_status_update", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + job_id = table.Column(type: "uuid", nullable: false), + at = table.Column(type: "timestamp with time zone", nullable: false), + status = table.Column(type: "integer", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_job_status_update", x => x.id); + table.ForeignKey( + name: "fk_job_status_update_jobs_job_id", + column: x => x.job_id, + principalTable: "jobs", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_job_status_update_job_id", + table: "job_status_update", + column: "job_id"); + + migrationBuilder.CreateIndex( + name: "ix_jobs_agent_id", + table: "jobs", + column: "agent_id"); + + migrationBuilder.CreateIndex( + name: "ix_jobs_source_id", + table: "jobs", + column: "source_id"); + + migrationBuilder.CreateIndex( + name: "ix_source_branches_source_id", + table: "source_branches", + column: "source_id"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "job_status_update"); + + migrationBuilder.DropTable( + name: "source_branches"); + + migrationBuilder.DropTable( + name: "jobs"); + + migrationBuilder.DropTable( + name: "agents"); + + migrationBuilder.DropTable( + name: "sources"); + } + } +} diff --git a/src/marlin.web/Migrations/20260905011100_AddCommitModel.Designer.cs b/src/marlin.web/Migrations/20260905011100_AddCommitModel.Designer.cs new file mode 100644 index 0000000..03f789e --- /dev/null +++ b/src/marlin.web/Migrations/20260905011100_AddCommitModel.Designer.cs @@ -0,0 +1,376 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using marlin.web.Data; + +#nullable disable + +namespace marlin.web.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20260905011100_AddCommitModel")] + partial class AddCommitModel + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("marlin.core.models.Agent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property("Commit") + .IsRequired() + .HasColumnType("text") + .HasColumnName("commit"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property("Ref") + .IsRequired() + .HasColumnType("text") + .HasColumnName("ref"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_update"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_update_job_id"); + + b.ToTable("job_status_update", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BranchFilter") + .IsRequired() + .HasColumnType("text") + .HasColumnName("branch_filter"); + + b.Property("MainBranch") + .IsRequired() + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + + b.Property("UpstreamUrl") + .IsRequired() + .HasColumnType("text") + .HasColumnName("upstream_url"); + + b.HasKey("Id") + .HasName("pk_sources"); + + b.ToTable("sources", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_blobs"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_blobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_blobs_source_id"); + + b.ToTable("source_blobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_branches_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BlobsFetched") + .HasColumnType("boolean") + .HasColumnName("blobs_fetched"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_commits"); + + b.HasAlternateKey("SourceId", "Hash") + .HasName("ak_source_commits_source_id_hash"); + + b.ToTable("source_commits", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.HasOne("marlin.core.models.Agent", "Agent") + .WithMany() + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.HasOne("marlin.core.models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_update_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Blobs") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Branches") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_commits_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Navigation("Blobs"); + + b.Navigation("Branches"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/marlin.web/Migrations/20260905011100_AddCommitModel.cs b/src/marlin.web/Migrations/20260905011100_AddCommitModel.cs new file mode 100644 index 0000000..78a23d0 --- /dev/null +++ b/src/marlin.web/Migrations/20260905011100_AddCommitModel.cs @@ -0,0 +1,126 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace marlin.web.Migrations +{ + /// + public partial class AddCommitModel : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "head_commit", + table: "source_branches"); + + migrationBuilder.AddColumn( + name: "commit_id", + table: "source_branches", + type: "uuid", + nullable: false, + defaultValue: new Guid("00000000-0000-0000-0000-000000000000")); + + migrationBuilder.CreateTable( + name: "source_commits", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + source_id = table.Column(type: "uuid", nullable: false), + blobs_fetched = table.Column(type: "boolean", nullable: false), + hash = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_source_commits", x => x.id); + table.UniqueConstraint("ak_source_commits_source_id_hash", x => new { x.source_id, x.hash }); + table.ForeignKey( + name: "fk_source_commits_sources_source_id", + column: x => x.source_id, + principalTable: "sources", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "source_blobs", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + source_id = table.Column(type: "uuid", nullable: false), + commit_id = table.Column(type: "uuid", nullable: false), + path = table.Column(type: "text", nullable: false), + content = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_source_blobs", x => x.id); + table.ForeignKey( + name: "fk_source_blobs_source_commits_commit_id", + column: x => x.commit_id, + principalTable: "source_commits", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "fk_source_blobs_sources_source_id", + column: x => x.source_id, + principalTable: "sources", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_source_branches_commit_id", + table: "source_branches", + column: "commit_id"); + + migrationBuilder.CreateIndex( + name: "ix_source_blobs_commit_id", + table: "source_blobs", + column: "commit_id"); + + migrationBuilder.CreateIndex( + name: "ix_source_blobs_source_id", + table: "source_blobs", + column: "source_id"); + + migrationBuilder.AddForeignKey( + name: "fk_source_branches_source_commits_commit_id", + table: "source_branches", + column: "commit_id", + principalTable: "source_commits", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "fk_source_branches_source_commits_commit_id", + table: "source_branches"); + + migrationBuilder.DropTable( + name: "source_blobs"); + + migrationBuilder.DropTable( + name: "source_commits"); + + migrationBuilder.DropIndex( + name: "ix_source_branches_commit_id", + table: "source_branches"); + + migrationBuilder.DropColumn( + name: "commit_id", + table: "source_branches"); + + migrationBuilder.AddColumn( + name: "head_commit", + table: "source_branches", + type: "text", + nullable: false, + defaultValue: ""); + } + } +} diff --git a/src/marlin.web/Migrations/20260905102259_AddSecretModel.Designer.cs b/src/marlin.web/Migrations/20260905102259_AddSecretModel.Designer.cs new file mode 100644 index 0000000..142703a --- /dev/null +++ b/src/marlin.web/Migrations/20260905102259_AddSecretModel.Designer.cs @@ -0,0 +1,430 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using marlin.web.Data; + +#nullable disable + +namespace marlin.web.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20260905102259_AddSecretModel")] + partial class AddSecretModel + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("marlin.core.models.Agent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property("Commit") + .IsRequired() + .HasColumnType("text") + .HasColumnName("commit"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property("Ref") + .IsRequired() + .HasColumnType("text") + .HasColumnName("ref"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_update"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_update_job_id"); + + b.ToTable("job_status_update", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccessSecretId") + .HasColumnType("uuid") + .HasColumnName("access_secret_id"); + + b.Property("BranchFilter") + .IsRequired() + .HasColumnType("text") + .HasColumnName("branch_filter"); + + b.Property("MainBranch") + .IsRequired() + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + + b.Property("UpstreamUrl") + .IsRequired() + .HasColumnType("text") + .HasColumnName("upstream_url"); + + b.HasKey("Id") + .HasName("pk_sources"); + + b.ToTable("sources", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_blobs"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_blobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_blobs_source_id"); + + b.ToTable("source_blobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_branches_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BlobsFetched") + .HasColumnType("boolean") + .HasColumnName("blobs_fetched"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_commits"); + + b.HasAlternateKey("SourceId", "Hash") + .HasName("ak_source_commits_source_id_hash"); + + b.ToTable("source_commits", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("Kind") + .HasColumnType("text") + .HasColumnName("kind"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("MayReveal") + .HasColumnType("boolean") + .HasColumnName("may_reveal"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_secrets"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_secrets_source_id"); + + b.ToTable("secrets", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.HasOne("marlin.core.models.Agent", "Agent") + .WithMany() + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.HasOne("marlin.core.models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_update_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Blobs") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Branches") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_commits_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_secrets_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Navigation("Blobs"); + + b.Navigation("Branches"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/marlin.web/Migrations/20260905102259_AddSecretModel.cs b/src/marlin.web/Migrations/20260905102259_AddSecretModel.cs new file mode 100644 index 0000000..8b09e89 --- /dev/null +++ b/src/marlin.web/Migrations/20260905102259_AddSecretModel.cs @@ -0,0 +1,59 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace marlin.web.Migrations +{ + /// + public partial class AddSecretModel : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "access_secret_id", + table: "sources", + type: "uuid", + nullable: true); + + migrationBuilder.CreateTable( + name: "secrets", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + source_id = table.Column(type: "uuid", nullable: false), + may_reveal = table.Column(type: "boolean", nullable: false), + kind = table.Column(type: "text", nullable: true), + label = table.Column(type: "text", nullable: false), + content = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_secrets", x => x.id); + table.ForeignKey( + name: "fk_secrets_sources_source_id", + column: x => x.source_id, + principalTable: "sources", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_secrets_source_id", + table: "secrets", + column: "source_id"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "secrets"); + + migrationBuilder.DropColumn( + name: "access_secret_id", + table: "sources"); + } + } +} diff --git a/src/marlin.web/Migrations/20260905104308_BlobNowBinary.Designer.cs b/src/marlin.web/Migrations/20260905104308_BlobNowBinary.Designer.cs new file mode 100644 index 0000000..6061a9d --- /dev/null +++ b/src/marlin.web/Migrations/20260905104308_BlobNowBinary.Designer.cs @@ -0,0 +1,430 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using marlin.web.Data; + +#nullable disable + +namespace marlin.web.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20260905104308_BlobNowBinary")] + partial class BlobNowBinary + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("marlin.core.models.Agent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property("Commit") + .IsRequired() + .HasColumnType("text") + .HasColumnName("commit"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property("Ref") + .IsRequired() + .HasColumnType("text") + .HasColumnName("ref"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_update"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_update_job_id"); + + b.ToTable("job_status_update", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccessSecretId") + .HasColumnType("uuid") + .HasColumnName("access_secret_id"); + + b.Property("BranchFilter") + .IsRequired() + .HasColumnType("text") + .HasColumnName("branch_filter"); + + b.Property("MainBranch") + .IsRequired() + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + + b.Property("UpstreamUrl") + .IsRequired() + .HasColumnType("text") + .HasColumnName("upstream_url"); + + b.HasKey("Id") + .HasName("pk_sources"); + + b.ToTable("sources", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_blobs"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_blobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_blobs_source_id"); + + b.ToTable("source_blobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_branches_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BlobsFetched") + .HasColumnType("boolean") + .HasColumnName("blobs_fetched"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_commits"); + + b.HasAlternateKey("SourceId", "Hash") + .HasName("ak_source_commits_source_id_hash"); + + b.ToTable("source_commits", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("Kind") + .HasColumnType("text") + .HasColumnName("kind"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("MayReveal") + .HasColumnType("boolean") + .HasColumnName("may_reveal"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_secrets"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_secrets_source_id"); + + b.ToTable("secrets", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.HasOne("marlin.core.models.Agent", "Agent") + .WithMany() + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.HasOne("marlin.core.models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_update_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Blobs") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Branches") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_commits_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_secrets_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Navigation("Blobs"); + + b.Navigation("Branches"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/marlin.web/Migrations/20260905104308_BlobNowBinary.cs b/src/marlin.web/Migrations/20260905104308_BlobNowBinary.cs new file mode 100644 index 0000000..493e698 --- /dev/null +++ b/src/marlin.web/Migrations/20260905104308_BlobNowBinary.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace marlin.web.Migrations +{ + /// + public partial class BlobNowBinary : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "content", + table: "source_blobs"); + + migrationBuilder.AddColumn( + name: "content", + table: "source_blobs", + type: "bytea", + nullable: false, + defaultValue: new byte[0]); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "content", + table: "source_blobs"); + + migrationBuilder.AddColumn( + name: "content", + table: "source_blobs", + type: "text", + nullable: false, + defaultValue: ""); + } + } +} diff --git a/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs b/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs index 4c026e5..8de35de 100644 --- a/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs @@ -22,18 +22,145 @@ namespace marlin.web.Migrations NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - modelBuilder.Entity("marlin.core.VersionControlSource", b => + modelBuilder.Entity("marlin.core.models.Agent", b => { b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("uuid") .HasColumnName("id"); + b.Property("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property("Commit") + .IsRequired() + .HasColumnType("text") + .HasColumnName("commit"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property("Ref") + .IsRequired() + .HasColumnType("text") + .HasColumnName("ref"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_update"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_update_job_id"); + + b.ToTable("job_status_update", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccessSecretId") + .HasColumnType("uuid") + .HasColumnName("access_secret_id"); + b.Property("BranchFilter") .IsRequired() .HasColumnType("text") .HasColumnName("branch_filter"); + b.Property("MainBranch") + .IsRequired() + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + b.Property("UpstreamUrl") .IsRequired() .HasColumnType("text") @@ -44,6 +171,256 @@ namespace marlin.web.Migrations b.ToTable("sources", (string)null); }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_blobs"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_blobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_blobs_source_id"); + + b.ToTable("source_blobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_branches_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BlobsFetched") + .HasColumnType("boolean") + .HasColumnName("blobs_fetched"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_commits"); + + b.HasAlternateKey("SourceId", "Hash") + .HasName("ak_source_commits_source_id_hash"); + + b.ToTable("source_commits", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("Kind") + .HasColumnType("text") + .HasColumnName("kind"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("MayReveal") + .HasColumnType("boolean") + .HasColumnName("may_reveal"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_secrets"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_secrets_source_id"); + + b.ToTable("secrets", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.HasOne("marlin.core.models.Agent", "Agent") + .WithMany() + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.HasOne("marlin.core.models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_update_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Blobs") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Branches") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_commits_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_secrets_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Navigation("Blobs"); + + b.Navigation("Branches"); + }); #pragma warning restore 612, 618 } } diff --git a/src/marlin.web/Models/DatabaseSourceSecretStore.cs b/src/marlin.web/Models/DatabaseSourceSecretStore.cs new file mode 100644 index 0000000..b4ca61f --- /dev/null +++ b/src/marlin.web/Models/DatabaseSourceSecretStore.cs @@ -0,0 +1,27 @@ +namespace marlin.web.Models; + +using core.models; +using Data; +using Microsoft.EntityFrameworkCore; + +public class DatabaseSourceSecretStore(ApplicationDbContext dbContext) : ISourceSecretStore +{ + public async Task GetAsync(Guid id, CancellationToken ct) + { + return await dbContext.Secrets + .FirstOrDefaultAsync(x => x.Id == id, cancellationToken: ct); + } +} + +public class VersionControlSecret : ISecret +{ + public Guid Id { get; set; } + + public Guid SourceId { get; set; } + public VersionControlSource Source { get; set; } + + public bool MayReveal { get; set; } + public string? Kind { get; set; } + public string Label { get; set; } + public Dictionary Content { get; set; } +} \ No newline at end of file diff --git a/src/marlin.web/Program.cs b/src/marlin.web/Program.cs index cf330b2..f079a3c 100644 --- a/src/marlin.web/Program.cs +++ b/src/marlin.web/Program.cs @@ -1,4 +1,5 @@ using marlin.web.Data; +using marlin.web.Models; using marlin.web.Services; using Microsoft.EntityFrameworkCore; @@ -10,8 +11,13 @@ builder.Services.AddDbContext(options => options .UseNpgsql(builder.Configuration.GetConnectionString("Master")) .UseSnakeCaseNamingConvention()); +builder.Services.AddScoped(); + builder.Services.AddHostedService(); -builder.Services.AddHostedService(); + +builder.Services.AddSingleton(); +builder.Services.AddHostedService(sp => sp + .GetRequiredService()); // Add services to the container. builder.Services.AddControllersWithViews(); diff --git a/src/marlin.web/Services/VersionControlSyncService.cs b/src/marlin.web/Services/VersionControlSyncService.cs index ba40662..bf189b0 100644 --- a/src/marlin.web/Services/VersionControlSyncService.cs +++ b/src/marlin.web/Services/VersionControlSyncService.cs @@ -4,8 +4,10 @@ using System.Diagnostics; using System.Text; using core.functional; using core.models; +using core.util; using Data; using Microsoft.EntityFrameworkCore; +using Models; public class VersionControlSyncService : BackgroundService { @@ -13,38 +15,213 @@ public class VersionControlSyncService : BackgroundService private readonly IServiceProvider _services; private ICollection _sources; - private Dictionary _nextPoll; + private Dictionary _lastPoll; public VersionControlSyncService(ILogger logger, IServiceProvider services) { this._logger = logger; this._services = services; this._sources = new List(); - this._nextPoll = new(); + this._lastPoll = new(); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await this.FetchSources(stoppingToken); + + while (!stoppingToken.IsCancellationRequested) + { + await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); + + try + { + await Tick(stoppingToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + this._logger.LogError(ex, "VCS sync tick failed"); + } + } } - public async Task FetchSources() + private async Task Tick(CancellationToken ct) { - await using var scoped = this._services.CreateAsyncScope(); - var dbContext = scoped.ServiceProvider.GetRequiredService(); - this._sources = await dbContext.Sources.ToListAsync(); + var now = DateTime.UtcNow; + var due = this + ._sources + .Where(x => this._lastPoll.TryGetValue(x.Id, out var lastTime) ? now >= lastTime.AddSeconds(x.MinPollingInterval) : true) + .ToArray(); + + if (due.Length > 0) + { + await using var scoped = this._services.CreateAsyncScope(); + var dbContext = scoped.ServiceProvider.GetRequiredService(); + var sourceSecretStore = scoped.ServiceProvider.GetRequiredService(); + + foreach (var source in due) + { + var trackedSource = await dbContext.Sources + .Include(x => x.Branches) + .ThenInclude(x => x.Commit) + .FirstAsync(x => x.Id == source.Id, ct); + + try + { + await this.SyncRefs(dbContext, sourceSecretStore, trackedSource, ct); + await this.SyncBlobs(dbContext, sourceSecretStore, trackedSource, ct); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + this._logger.LogError(ex, "Failed to sync vcs source '{name}'", source.UpstreamUrl); + } + finally + { + this._lastPoll[source.Id] = DateTime.UtcNow; + } + } + + if (dbContext.ChangeTracker.HasChanges()) + { + await this.Sweep(dbContext, ct); + + await dbContext.SaveChangesAsync(ct); + this._logger.LogInformation("Synced {count} vcs roots", due.Length); + } + } } - private async Task SyncSource(VersionControlSource source, ApplicationDbContext db) + private async Task GetClient(VersionControlSource source, ISourceSecretStore store, CancellationToken ct) + { + IGitCredential? cred = null; + if (source.AccessSecretId != null) + { + ISecret? secret = await store.GetAsync(source.AccessSecretId.Value, ct); + if (secret == null) + throw new InvalidOperationException($"Could not find access secret for '{source.UpstreamUrl}'"); + + if (secret.Kind == SecretKind.SSHKey && secret.Has("public_key", "private_key")) + cred = new SshKeyGitCredential(secret.Content["private_key"]); + + else if (secret.Kind == SecretKind.HTTPToken && secret.Has("identity", "token")) + cred = new TokenGitCredential(secret.Content["token"], secret.Content["identity"]); + + else + throw new InvalidOperationException($"Could not resolve access secret content to credentials for '{source.UpstreamUrl}'"); + } + + IGitClient client = new GitBinaryClient(cred); + return client; + } + + private async Task GetLocalPath(IGitClient client, VersionControlSource source) + { + var dir = Path.Join(Environment.CurrentDirectory, "run", "sources", source.Id.ToString("N")); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + await client.Init(dir); + } + + return dir; + } + + public async Task FetchSources(CancellationToken ct) { - IGitClient client = new GitBinaryClient(); - var refs = await client.GetRemoteRefs(source.UpstreamUrl); + await using var scoped = this._services.CreateAsyncScope(); + var dbContext = scoped.ServiceProvider.GetRequiredService(); + this._sources = await dbContext.Sources + .Include(x => x.Branches) + .ToListAsync(ct); + await this.Sweep(dbContext, ct); } - protected override async Task ExecuteAsync(CancellationToken stoppingToken) + private async Task Sweep(ApplicationDbContext dbContext, CancellationToken ct) { - await this.FetchSources(); + await dbContext.SourceCommits + .Include(x => x.Branches) + .Where(x => !x.Branches.Any()) + .ExecuteDeleteAsync(ct); + } + + private async Task SyncRefs(ApplicationDbContext dbContext, ISourceSecretStore secretStore, VersionControlSource source, CancellationToken ct) + { + IGitClient client = await this.GetClient(source, secretStore, ct); + var newRefs = await client.GetRemoteRefs(source.UpstreamUrl); + + Dictionary currentRefs = source.Branches + .ToDictionary(x => x.Name, x => x.Commit.Hash); + Dictionary branchesByName = source.Branches + .ToDictionary(x => x.Name); + + var added = newRefs.Keys.Except(currentRefs.Keys).ToArray(); + var removed = currentRefs.Keys.Except(newRefs.Keys).ToArray(); + var changed = currentRefs.Keys.Intersect(newRefs.Keys) + .Where(name => currentRefs[name] != newRefs[name]) + .ToArray(); + + foreach (var name in removed) + source.Branches.Remove(branchesByName[name]); + + // Pre-load the commits these refs now point to, so a hash shared across + // multiple branches (or seen in an earlier sync) resolves to one row + // instead of racing duplicate inserts. + var neededHashes = added.Concat(changed) + .Select(name => newRefs[name]) + .Distinct() + .ToHashSet(); + + var commitsByHash = await dbContext.SourceCommits + .Where(c => c.SourceId == source.Id && neededHashes.Contains(c.Hash)) + .ToDictionaryAsync(c => c.Hash, ct); + + VersionControlledCommit GetOrCreateCommit(string hash) + { + if (!commitsByHash.TryGetValue(hash, out var commit)) + { + commit = new VersionControlledCommit { SourceId = source.Id, Hash = hash, BlobsFetched = false }; + commitsByHash[hash] = commit; + } + + return commit; + } + + foreach (var name in added) + source.Branches.Add(new VersionControlledBranch { Name = name, Commit = GetOrCreateCommit(newRefs[name]) }); + + foreach (var name in changed) + branchesByName[name].Commit = GetOrCreateCommit(newRefs[name]); + } + + private async Task SyncBlobs(ApplicationDbContext dbContext, ISourceSecretStore secretStore, VersionControlSource source, CancellationToken ct) + { + var refs = source.Branches + .Where(x => !x.Commit.BlobsFetched) + .DistinctBy(x => x.CommitId) + .ToArray(); + + IGitClient client = await this.GetClient(source, secretStore, ct); + var localPath = await GetLocalPath(client, source); + await client.FetchTrees(localPath, source.UpstreamUrl, refs.Select(x => x.Name)); - while (!stoppingToken.IsCancellationRequested) + foreach (VersionControlledCommit commit in refs.Select(x => x.Commit)) { + var blobNames = await client.ListTree(localPath, commit.Hash, "."); + + foreach (string blobName in blobNames) + { + var content = await client.CatFile(localPath, commit.Hash, blobName); + dbContext.SourceBlobs.Add(new VersionControlledBlob + { + Source = source, + Commit = commit, + Path = blobName, + Content = content + }); + } + commit.BlobsFetched = true; } } } \ No newline at end of file diff --git a/src/marlin.web/Views/Jobs/Details.cshtml b/src/marlin.web/Views/Jobs/Details.cshtml new file mode 100644 index 0000000..e1fd184 --- /dev/null +++ b/src/marlin.web/Views/Jobs/Details.cshtml @@ -0,0 +1,5 @@ +@model marlin.core.models.Job + +@Model.Id + +Back \ No newline at end of file diff --git a/src/marlin.web/Views/Jobs/List.cshtml b/src/marlin.web/Views/Jobs/List.cshtml index 2c10e46..6ae1d96 100644 --- a/src/marlin.web/Views/Jobs/List.cshtml +++ b/src/marlin.web/Views/Jobs/List.cshtml @@ -1,5 +1,7 @@ @model ICollection +Spawn new... + @@ -10,6 +12,7 @@ + @@ -23,6 +26,7 @@ + } diff --git a/src/marlin.web/Views/Jobs/Spawn.cshtml b/src/marlin.web/Views/Jobs/Spawn.cshtml new file mode 100644 index 0000000..8f7ee43 --- /dev/null +++ b/src/marlin.web/Views/Jobs/Spawn.cshtml @@ -0,0 +1,56 @@ +@using System.Linq +@using System.Text.Json +@using marlin.web.Controllers +@model marlin.web.Controllers.JobsController.JobSpawnDetails + +@{ + var sourcesJson = JsonSerializer.Serialize(Model.Sources.Select(s => new + { + id = s.Id, + label = s.UpstreamUrl, + mainBranch = s.MainBranch, + branches = s.Branches.Select(b => new { name = b.Name, commit = b.Commit.Hash }) + })); +} + + + VCS:
+ Ref:
+ Commit:
+ Recipe:
+ + + + diff --git a/src/marlin.web/Views/Shared/_Layout.cshtml b/src/marlin.web/Views/Shared/_Layout.cshtml index f060fcf..e8be2af 100644 --- a/src/marlin.web/Views/Shared/_Layout.cshtml +++ b/src/marlin.web/Views/Shared/_Layout.cshtml @@ -24,7 +24,10 @@ Home
  • - Privacy + VCS Configuration +
    • +
  • + Jobs
    • diff --git a/src/marlin.web/Views/VCS/Create.cshtml b/src/marlin.web/Views/VCS/Create.cshtml index adaa913..e04592d 100644 --- a/src/marlin.web/Views/VCS/Create.cshtml +++ b/src/marlin.web/Views/VCS/Create.cshtml @@ -2,10 +2,10 @@ @using marlin.core.models - URL: - Filter: - Main Branch: - Polling Interval: + URL:
    + Filter:
    + Main Branch:
    + Polling Interval:
    diff --git a/src/marlin.web/Views/VCS/CreateSecret.cshtml b/src/marlin.web/Views/VCS/CreateSecret.cshtml new file mode 100644 index 0000000..6453056 --- /dev/null +++ b/src/marlin.web/Views/VCS/CreateSecret.cshtml @@ -0,0 +1,12 @@ +@using marlin.core.models +@model VersionControlSource + + + Label:
    + Kind:
    + May Reveal:
    + Content (JSON):
    + + + +Back diff --git a/src/marlin.web/Views/VCS/Details.cshtml b/src/marlin.web/Views/VCS/Details.cshtml index 8d25472..45c1a8f 100644 --- a/src/marlin.web/Views/VCS/Details.cshtml +++ b/src/marlin.web/Views/VCS/Details.cshtml @@ -1,13 +1,43 @@ @using marlin.core @using marlin.core.models -@model marlin.core.models.VersionControlSource +@using marlin.web.Controllers +@model (VersionControlSource Source, List SecretOptions) -
    - URL: - Filter: - Main Branch: - Polling Interval: + + URL:
    + Filter:
    + Main Branch:
    + Polling Interval:
    + Access Secret: +
    -Back \ No newline at end of file +
    vcs branch agentcreated
    @source.Source.UpstreamUrl @source.Ref @(source.Agent?.Name ?? "no agent assigned")@(source.CreatedReason)
    + + + + + + + + + @foreach (var branch in Model.Source.Branches) + { + + + + + + } + +
    branchcommit
    @(branch.Name == Model.Source.MainBranch ? "@" : "")@branch.Name@branch.Commit.Hash
    + +Secrets +Back diff --git a/src/marlin.web/Views/VCS/List.cshtml b/src/marlin.web/Views/VCS/List.cshtml index 90c3f48..4c248e4 100644 --- a/src/marlin.web/Views/VCS/List.cshtml +++ b/src/marlin.web/Views/VCS/List.cshtml @@ -1,4 +1,4 @@ -@model IReadOnlyList +@model ICollection New diff --git a/src/marlin.web/Views/VCS/ListSecret.cshtml b/src/marlin.web/Views/VCS/ListSecret.cshtml new file mode 100644 index 0000000..aa5f15c --- /dev/null +++ b/src/marlin.web/Views/VCS/ListSecret.cshtml @@ -0,0 +1,45 @@ +@model (marlin.core.models.VersionControlSource Source, List Secrets) + +New + + + + + + + + + + + + + @foreach (var secret in Model.Secrets) + { + + + + + + + + } + +
    idlabelkindvalueactions
    @secret.Id@secret.Label@secret.Kind + @if (secret.MayReveal) + { + @foreach (var pair in secret.Content) + { + @pair.Key = @pair.Value + } + } + else + { + secret cannot be revealed + } + +
    + +
    +
    + +back diff --git a/src/marlin.web/appsettings.json b/src/marlin.web/appsettings.json index 10f68b8..793d7e4 100644 --- a/src/marlin.web/appsettings.json +++ b/src/marlin.web/appsettings.json @@ -2,7 +2,8 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Warning", + "Microsoft.EntityFrameworkCore.Database.Command": "Warning" } }, "AllowedHosts": "*" diff --git a/src/marlin.web/marlin.web.csproj b/src/marlin.web/marlin.web.csproj index 3160c98..26c8ea3 100644 --- a/src/marlin.web/marlin.web.csproj +++ b/src/marlin.web/marlin.web.csproj @@ -21,4 +21,8 @@ + + + + diff --git a/src/marlin.web/wwwroot/css/site.css b/src/marlin.web/wwwroot/css/site.css index 819f612..c6ecfb7 100644 --- a/src/marlin.web/wwwroot/css/site.css +++ b/src/marlin.web/wwwroot/css/site.css @@ -28,4 +28,24 @@ body { .form-floating > .form-control-plaintext:focus::placeholder, .form-floating > .form-control:focus::placeholder { text-align: start; +} + +table { + border-collapse: collapse; + width: 100%; +} + +th, td { + border: 1px solid #ccc; + text-align: left; +} + +th { + background-color: #f0f0f0; + font-weight: bold; +} + +th a { + text-decoration: none; + color: inherit; } \ No newline at end of file -- cgit v1.3.1