From df0737e55afeb5c01eb39909605970d1d3a73ffc Mon Sep 17 00:00:00 2001 From: Melodi <62381002+Melodi17@users.noreply.github.com> Date: Sun, 13 Sep 2026 13:22:20 +1000 Subject: refactor: cleanup; feat: connection to agent --- src/marlin.agents.shell/MarlinApiClient.cs | 38 ++ src/marlin.agents.shell/Program.cs | 21 +- src/marlin.agents.shell/marlin.agents.shell.csproj | 4 + src/marlin.core/functional/WorkflowBuilder.cs | 153 ++++++ src/marlin.core/functional/WorkflowJson.cs | 17 + src/marlin.core/marlin.core.csproj | 4 + src/marlin.core/models/Agent.cs | 12 - src/marlin.core/models/AgentStatus.cs | 4 + src/marlin.core/models/CapabilitySet.cs | 7 + src/marlin.core/models/DispatchJob.cs | 9 + src/marlin.core/models/Job.cs | 42 -- src/marlin.core/models/SecretKind.cs | 7 - src/marlin.core/models/VersionControlSource.cs | 15 - src/marlin.core/models/VersionControlledBlob.cs | 15 - src/marlin.core/models/VersionControlledBranch.cs | 14 - src/marlin.core/models/VersionControlledCommit.cs | 15 - src/marlin.core/models/meta/AbstractStep.cs | 16 + src/marlin.core/models/meta/Artifact.cs | 26 ++ src/marlin.core/models/meta/ArtifactHandler.cs | 12 + src/marlin.core/models/meta/ArtifactHandlerStep.cs | 14 + src/marlin.core/models/meta/CleanupAfter.cs | 7 + src/marlin.core/models/meta/CleanupStep.cs | 14 + src/marlin.core/models/meta/FailBehavior.cs | 8 + src/marlin.core/models/meta/Recipe.cs | 36 ++ src/marlin.core/models/meta/RecipeStep.cs | 28 ++ src/marlin.core/models/meta/RequirementSet.cs | 13 + src/marlin.core/models/meta/Workflow.cs | 9 + src/marlin.core/models/meta/dto/ArtifactDto.cs | 8 + .../models/meta/dto/ArtifactHandlerDto.cs | 6 + src/marlin.core/models/meta/dto/CleanupStepDto.cs | 6 + src/marlin.core/models/meta/dto/RecipeDto.cs | 11 + .../models/meta/dto/RequirementSetDto.cs | 7 + src/marlin.core/models/meta/dto/StepDto.cs | 9 + src/marlin.core/models/meta/dto/StepDtoBase.cs | 9 + src/marlin.core/models/meta/dto/WorkflowFileDto.cs | 9 + src/marlin.core/models/plan/CommandResult.cs | 17 + .../models/plan/CommandResultSelector.cs | 13 + src/marlin.core/models/plan/CommandRole.cs | 8 + src/marlin.core/models/plan/ExecutionCommand.cs | 15 + src/marlin.core/models/plan/ExecutionPlan.cs | 174 +++++++ src/marlin.core/util/SecureTokenGenerator.cs | 16 + src/marlin.core/util/VariableExpander.cs | 140 ++++++ src/marlin.core/util/VariableExpansionException.cs | 56 +++ src/marlin.web/Controllers/AgentsController.cs | 89 ++++ src/marlin.web/Controllers/DispatchController.cs | 65 ++- src/marlin.web/Controllers/JobsController.cs | 35 +- src/marlin.web/Controllers/VCSController.cs | 36 +- src/marlin.web/Data/ApplicationDbContext.cs | 23 +- src/marlin.web/Data/JsonValueComparer.cs | 23 + .../20260906003038_AddWorkflowSnapshot.Designer.cs | 478 +++++++++++++++++++ .../20260906003038_AddWorkflowSnapshot.cs | 59 +++ .../20260906082039_AddAgentToken.Designer.cs | 488 +++++++++++++++++++ .../Migrations/20260906082039_AddAgentToken.cs | 29 ++ .../20260908142748_CommitAlt.Designer.cs | 514 +++++++++++++++++++++ .../Migrations/20260908142748_CommitAlt.cs | 115 +++++ .../ApplicationDbContextModelSnapshot.cs | 204 +++++--- src/marlin.web/Models/Agent.cs | 17 + src/marlin.web/Models/Job.cs | 48 ++ src/marlin.web/Models/SecretKind.cs | 7 + src/marlin.web/Models/VersionControlSource.cs | 15 + src/marlin.web/Models/VersionControlledBlob.cs | 15 + src/marlin.web/Models/VersionControlledBranch.cs | 14 + src/marlin.web/Models/VersionControlledCommit.cs | 17 + src/marlin.web/Models/WorkflowSnapshot.cs | 14 + src/marlin.web/Program.cs | 14 +- src/marlin.web/Properties/launchSettings.json | 4 +- .../Services/AgentAuthenticationHandler.cs | 79 ++++ .../Services/VersionControlSyncService.cs | 38 +- src/marlin.web/Util/ClaimsPrincipalExtensions.cs | 22 + src/marlin.web/Views/Agents/Create.cshtml | 9 + src/marlin.web/Views/Agents/Details.cshtml | 67 +++ src/marlin.web/Views/Agents/List.cshtml | 32 ++ src/marlin.web/Views/Jobs/Details.cshtml | 2 +- src/marlin.web/Views/Jobs/List.cshtml | 8 +- src/marlin.web/Views/Jobs/Spawn.cshtml | 37 +- src/marlin.web/Views/Shared/_Layout.cshtml | 3 + src/marlin.web/Views/VCS/Blobs.cshtml | 22 + src/marlin.web/Views/VCS/Details.cshtml | 12 + src/marlin.web/Views/VCS/List.cshtml | 2 +- src/marlin.web/Views/VCS/ListSecret.cshtml | 2 +- src/marlin.web/Views/VCS/Workflow.cshtml | 18 + src/marlin.web/wwwroot/css/site.css | 31 ++ 82 files changed, 3542 insertions(+), 229 deletions(-) create mode 100644 src/marlin.agents.shell/MarlinApiClient.cs create mode 100644 src/marlin.core/functional/WorkflowBuilder.cs create mode 100644 src/marlin.core/functional/WorkflowJson.cs delete mode 100644 src/marlin.core/models/Agent.cs create mode 100644 src/marlin.core/models/AgentStatus.cs create mode 100644 src/marlin.core/models/DispatchJob.cs delete mode 100644 src/marlin.core/models/Job.cs delete mode 100644 src/marlin.core/models/SecretKind.cs delete mode 100644 src/marlin.core/models/VersionControlSource.cs delete mode 100644 src/marlin.core/models/VersionControlledBlob.cs delete mode 100644 src/marlin.core/models/VersionControlledBranch.cs delete mode 100644 src/marlin.core/models/VersionControlledCommit.cs create mode 100644 src/marlin.core/models/meta/AbstractStep.cs create mode 100644 src/marlin.core/models/meta/Artifact.cs create mode 100644 src/marlin.core/models/meta/ArtifactHandler.cs create mode 100644 src/marlin.core/models/meta/ArtifactHandlerStep.cs create mode 100644 src/marlin.core/models/meta/CleanupAfter.cs create mode 100644 src/marlin.core/models/meta/CleanupStep.cs create mode 100644 src/marlin.core/models/meta/FailBehavior.cs create mode 100644 src/marlin.core/models/meta/Recipe.cs create mode 100644 src/marlin.core/models/meta/RecipeStep.cs create mode 100644 src/marlin.core/models/meta/RequirementSet.cs create mode 100644 src/marlin.core/models/meta/Workflow.cs create mode 100644 src/marlin.core/models/meta/dto/ArtifactDto.cs create mode 100644 src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs create mode 100644 src/marlin.core/models/meta/dto/CleanupStepDto.cs create mode 100644 src/marlin.core/models/meta/dto/RecipeDto.cs create mode 100644 src/marlin.core/models/meta/dto/RequirementSetDto.cs create mode 100644 src/marlin.core/models/meta/dto/StepDto.cs create mode 100644 src/marlin.core/models/meta/dto/StepDtoBase.cs create mode 100644 src/marlin.core/models/meta/dto/WorkflowFileDto.cs create mode 100644 src/marlin.core/models/plan/CommandResult.cs create mode 100644 src/marlin.core/models/plan/CommandResultSelector.cs create mode 100644 src/marlin.core/models/plan/CommandRole.cs create mode 100644 src/marlin.core/models/plan/ExecutionCommand.cs create mode 100644 src/marlin.core/models/plan/ExecutionPlan.cs create mode 100644 src/marlin.core/util/SecureTokenGenerator.cs create mode 100644 src/marlin.core/util/VariableExpander.cs create mode 100644 src/marlin.core/util/VariableExpansionException.cs create mode 100644 src/marlin.web/Controllers/AgentsController.cs create mode 100644 src/marlin.web/Data/JsonValueComparer.cs create mode 100644 src/marlin.web/Migrations/20260906003038_AddWorkflowSnapshot.Designer.cs create mode 100644 src/marlin.web/Migrations/20260906003038_AddWorkflowSnapshot.cs create mode 100644 src/marlin.web/Migrations/20260906082039_AddAgentToken.Designer.cs create mode 100644 src/marlin.web/Migrations/20260906082039_AddAgentToken.cs create mode 100644 src/marlin.web/Migrations/20260908142748_CommitAlt.Designer.cs create mode 100644 src/marlin.web/Migrations/20260908142748_CommitAlt.cs create mode 100644 src/marlin.web/Models/Agent.cs create mode 100644 src/marlin.web/Models/Job.cs create mode 100644 src/marlin.web/Models/SecretKind.cs create mode 100644 src/marlin.web/Models/VersionControlSource.cs create mode 100644 src/marlin.web/Models/VersionControlledBlob.cs create mode 100644 src/marlin.web/Models/VersionControlledBranch.cs create mode 100644 src/marlin.web/Models/VersionControlledCommit.cs create mode 100644 src/marlin.web/Models/WorkflowSnapshot.cs create mode 100644 src/marlin.web/Services/AgentAuthenticationHandler.cs create mode 100644 src/marlin.web/Util/ClaimsPrincipalExtensions.cs create mode 100644 src/marlin.web/Views/Agents/Create.cshtml create mode 100644 src/marlin.web/Views/Agents/Details.cshtml create mode 100644 src/marlin.web/Views/Agents/List.cshtml create mode 100644 src/marlin.web/Views/VCS/Blobs.cshtml create mode 100644 src/marlin.web/Views/VCS/Workflow.cshtml (limited to 'src') diff --git a/src/marlin.agents.shell/MarlinApiClient.cs b/src/marlin.agents.shell/MarlinApiClient.cs new file mode 100644 index 0000000..aa4f013 --- /dev/null +++ b/src/marlin.agents.shell/MarlinApiClient.cs @@ -0,0 +1,38 @@ +namespace marlin.agents.shell; + +using System.Net.Http.Json; +using core.models; + +public class MarlinApiClient +{ + private readonly HttpClient _http; + + public MarlinApiClient(string baseUrl, Guid agentId, string agentToken) + { + _http = new HttpClient { BaseAddress = new Uri(baseUrl) }; + _http.DefaultRequestHeaders.Add("Agent", $"{agentId} {agentToken}"); + } + + public static MarlinApiClient FromEnvironment() + { + var baseUrl = RequireEnv("MARLIN_API_BASE_URL"); + var agentId = RequireEnv("MARLIN_AGENT_ID"); + var agentToken = RequireEnv("MARLIN_AGENT_TOKEN"); + + return new MarlinApiClient(baseUrl, Guid.Parse(agentId), agentToken); + } + + private static string RequireEnv(string name) => + Environment.GetEnvironmentVariable(name) + ?? throw new InvalidOperationException($"Missing required environment variable: {name}"); + + public async Task Handshake(AgentStatus status, CancellationToken ct = default) + { + var response = await _http.PostAsJsonAsync("api/dispatch/handshake", status, ct); + response.EnsureSuccessStatusCode(); + return (await response.Content.ReadFromJsonAsync(ct))!; + } + + public async Task QueueNextJob(CancellationToken ct = default) + => await _http.GetFromJsonAsync("api/dispatch/queue", ct); +} diff --git a/src/marlin.agents.shell/Program.cs b/src/marlin.agents.shell/Program.cs index 837131c..02a0bc9 100644 --- a/src/marlin.agents.shell/Program.cs +++ b/src/marlin.agents.shell/Program.cs @@ -1 +1,20 @@ -Console.WriteLine("Hello, World!"); \ No newline at end of file +using System.Runtime.InteropServices; +using marlin.agents.shell; +using marlin.core.models; + +DotNetEnv.Env.NoClobber().TraversePath().Load(); + +var client = MarlinApiClient.FromEnvironment(); + +var capabilities = new CapabilitySet(); +capabilities.Items.Add(KnownCapabilities.Architecture, RuntimeInformation.OSArchitecture.ToString()); +capabilities.Items.Add(KnownCapabilities.CoreCount, Environment.ProcessorCount.ToString()); +capabilities.Items.Add(KnownCapabilities.Memory, (GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / (1024 * 1024)).ToString()); +var status = new AgentStatus(capabilities); + +AgentConfiguration config = await client.Handshake(status); +Console.WriteLine($"IAM: {config.Name}"); +Console.WriteLine($"MCJ: {config.MaxConcurrentJobs}"); + +var job = await client.QueueNextJob(); +Console.WriteLine(job == null ? "No pending jobs." : $"Claimed job {job.Id}: {job.Recipe}"); diff --git a/src/marlin.agents.shell/marlin.agents.shell.csproj b/src/marlin.agents.shell/marlin.agents.shell.csproj index cd0016d..ff86d63 100644 --- a/src/marlin.agents.shell/marlin.agents.shell.csproj +++ b/src/marlin.agents.shell/marlin.agents.shell.csproj @@ -7,6 +7,10 @@ enable + + + + diff --git a/src/marlin.core/functional/WorkflowBuilder.cs b/src/marlin.core/functional/WorkflowBuilder.cs new file mode 100644 index 0000000..79da909 --- /dev/null +++ b/src/marlin.core/functional/WorkflowBuilder.cs @@ -0,0 +1,153 @@ +namespace marlin.core.functional; + +using models.meta; +using models.meta.dto; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +public class WorkflowBuilder +{ + private static readonly IDeserializer Deserializer = new DeserializerBuilder() + .WithNamingConvention(HyphenatedNamingConvention.Instance) + .WithEnumNamingConvention(HyphenatedNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + private readonly Workflow _workflow = new(); + + public Workflow Build() => this._workflow; + + public void LoadFromDirectory(string path) + { + var files = Directory.EnumerateFiles(path, "*.yml", SearchOption.AllDirectories); + + foreach (var file in files) + this.LoadFromString(File.ReadAllText(file)); + } + + public void LoadFromString(string content) + { + var file = Deserializer.Deserialize(content); + if (file == null) + return; + + this.ApplyRequirements(file.Requirements); + this.ApplyRecipes(file.Recipes); + this.ApplySteps(file.Steps); + this.ApplyArtifactHandlers(file.ArtifactHandlers); + } + + private void ApplyRequirements(Dictionary? requirements) + { + if (requirements == null) + return; + + foreach (var (id, dto) in requirements) + { + var requirementSet = new RequirementSet(id) + { + Base = dto.Base, + Has = dto.Has != null ? new Dictionary(dto.Has) : new() + }; + + this._workflow.Requirements[id] = requirementSet; + } + } + + private void ApplyRecipes(Dictionary? recipes) + { + if (recipes == null) + return; + + foreach (var (id, dto) in recipes) + { + var recipe = new Recipe(id) + { + Name = dto.Name ?? id, + RunsOnId = dto.RunsOn, + RequireIds = dto.Requires ?? [], + Parameters = dto.Parameters != null ? new Dictionary(dto.Parameters) : new(), + Environment = dto.Environment != null ? new Dictionary(dto.Environment) : new() + }; + + if (dto.Steps != null) + { + foreach (var (stepId, stepDto) in dto.Steps) + recipe.Steps[stepId] = this.BuildRecipeStep(stepId, stepDto); + } + + this._workflow.Recipes[id] = recipe; + } + } + + private void ApplySteps(Dictionary? steps) + { + if (steps == null) + return; + + foreach (var (id, dto) in steps) + this._workflow.Steps[id] = this.BuildRecipeStep(id, dto); + } + + private RecipeStep BuildRecipeStep(string id, StepDto dto) + { + var step = new RecipeStep(id); + ApplyAbstractStep(step, dto); + + step.BaseId = dto.Base; + step.Ingest = dto.Ingest ?? []; + + if (dto.Artifacts != null) + { + foreach (var (artifactId, artifactDto) in dto.Artifacts) + { + step.Artifacts[artifactId] = new Artifact(artifactId) + { + Path = artifactDto.Path!, + Type = artifactDto.As!, + Handle = artifactDto.Handle ?? ArtifactHandle.Defer + }; + } + } + + if (dto.Cleanup != null) + step.Cleanup = BuildCleanupStep(id + "/cleanup", dto.Cleanup); + + return step; + } + + private static CleanupStep BuildCleanupStep(string id, CleanupStepDto dto) + { + var step = new CleanupStep(id) { After = dto.After ?? CleanupAfter.Step }; + ApplyAbstractStep(step, dto); + return step; + } + + private void ApplyArtifactHandlers(Dictionary? handlers) + { + if (handlers == null) + return; + + foreach (var (id, dto) in handlers) + { + var handler = new ArtifactHandler(id); + + if (dto.Handle != null) + { + var handleStep = new ArtifactHandlerStep(id + "/handle"); + ApplyAbstractStep(handleStep, dto.Handle); + handler.Handle = handleStep; + } + + this._workflow.ArtifactHandlers[id] = handler; + } + } + + private static void ApplyAbstractStep(AbstractStep step, StepDtoBase dto) + { + step.Name = dto.Name; + step.Run = dto.Run; + step.With = dto.With != null ? new Dictionary(dto.With) : new(); + step.Fail = dto.Fail; + } +} diff --git a/src/marlin.core/functional/WorkflowJson.cs b/src/marlin.core/functional/WorkflowJson.cs new file mode 100644 index 0000000..011edae --- /dev/null +++ b/src/marlin.core/functional/WorkflowJson.cs @@ -0,0 +1,17 @@ +namespace marlin.core.functional; + +using System.Text.Json; +using System.Text.Json.Serialization; +using models.meta; + +public static class WorkflowJson +{ + private static readonly JsonSerializerOptions Options = new() + { + Converters = { new JsonStringEnumConverter() } + }; + + public static string Serialize(Workflow workflow) => JsonSerializer.Serialize(workflow, Options); + + public static Workflow Deserialize(string json) => JsonSerializer.Deserialize(json, Options)!; +} diff --git a/src/marlin.core/marlin.core.csproj b/src/marlin.core/marlin.core.csproj index 237d661..5b65fba 100644 --- a/src/marlin.core/marlin.core.csproj +++ b/src/marlin.core/marlin.core.csproj @@ -6,4 +6,8 @@ enable + + + + diff --git a/src/marlin.core/models/Agent.cs b/src/marlin.core/models/Agent.cs deleted file mode 100644 index 2903131..0000000 --- a/src/marlin.core/models/Agent.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace marlin.core.models; - -public class Agent -{ - public Guid Id { get; set; } - - public string Name { get; set; } - public int MaxConcurrentJobs { get; set; } - public DateTime LastSeen { get; set; } - - public CapabilitySet Capabilities { get; set; } -} \ No newline at end of file diff --git a/src/marlin.core/models/AgentStatus.cs b/src/marlin.core/models/AgentStatus.cs new file mode 100644 index 0000000..393062f --- /dev/null +++ b/src/marlin.core/models/AgentStatus.cs @@ -0,0 +1,4 @@ +namespace marlin.core.models; + +public record AgentStatus(CapabilitySet CapabilitySet); +public record AgentConfiguration(string Name, int MaxConcurrentJobs); \ No newline at end of file diff --git a/src/marlin.core/models/CapabilitySet.cs b/src/marlin.core/models/CapabilitySet.cs index ac7da46..553d746 100644 --- a/src/marlin.core/models/CapabilitySet.cs +++ b/src/marlin.core/models/CapabilitySet.cs @@ -3,4 +3,11 @@ namespace marlin.core.models; public class CapabilitySet { public Dictionary Items { get; set; } = new(); +} + +public static class KnownCapabilities +{ + public const string CoreCount = "cores"; + public const string Memory = "memory"; + public const string Architecture = "architecture"; } \ No newline at end of file diff --git a/src/marlin.core/models/DispatchJob.cs b/src/marlin.core/models/DispatchJob.cs new file mode 100644 index 0000000..4070447 --- /dev/null +++ b/src/marlin.core/models/DispatchJob.cs @@ -0,0 +1,9 @@ +namespace marlin.core.models; + +public record DispatchJob( + Guid Id, + string Token, + string Vcs, + string Ref, + string Commit, + string Recipe); diff --git a/src/marlin.core/models/Job.cs b/src/marlin.core/models/Job.cs deleted file mode 100644 index a8c7c5d..0000000 --- a/src/marlin.core/models/Job.cs +++ /dev/null @@ -1,42 +0,0 @@ -namespace marlin.core.models; - -public class Job -{ - public Guid Id { get; set; } - - public Guid SourceId { get; set; } - public VersionControlSource Source { get; set; } - public string Ref { get; set; } - public string Commit { get; set; } - - public string CreatedReason { get; set; } - public DateTime CreatedAt { get; set; } - public JobStatus Status { get; set; } - public ICollection StatusUpdates { get; set; } - - public Guid? AgentId { get; set; } - public Agent? Agent { get; set; } - - public string Recipe { get; set; } - // plan -} - -public class JobStatusUpdate -{ - public Guid Id { get; set; } - - public Guid JobId { get; set; } - public Job Job { get; set; } - - public DateTime At { get; set; } - public JobStatus Status { get; set; } -} - -public enum JobStatus -{ - Pending, - Active, - Cancelled, - Success, - Failed -} \ No newline at end of file diff --git a/src/marlin.core/models/SecretKind.cs b/src/marlin.core/models/SecretKind.cs deleted file mode 100644 index 02652d5..0000000 --- a/src/marlin.core/models/SecretKind.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace marlin.web.Models; - -public static class SecretKind -{ - public const string HTTPToken = "http-token"; - public const string SSHKey = "ssh-key"; -} \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlSource.cs b/src/marlin.core/models/VersionControlSource.cs deleted file mode 100644 index daa8d51..0000000 --- a/src/marlin.core/models/VersionControlSource.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace marlin.core.models; - -public class VersionControlSource -{ - public Guid Id { get; set; } - - public string UpstreamUrl { get; set; } - public string MainBranch { get; set; } - public string BranchFilter { get; set; } - public int MinPollingInterval { get; set; } - - public Guid? AccessSecretId { get; set; } - - public ICollection Branches { get; set; } -} \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlledBlob.cs b/src/marlin.core/models/VersionControlledBlob.cs deleted file mode 100644 index 03915be..0000000 --- a/src/marlin.core/models/VersionControlledBlob.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace marlin.core.models; - -public class VersionControlledBlob -{ - public Guid Id { get; set; } - - public Guid SourceId { get; set; } - public VersionControlSource Source { get; set; } - - public Guid CommitId { get; set; } - public VersionControlledCommit Commit { get; set; } - - public string Path { get; set; } - public byte[] Content { get; set; } -} \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlledBranch.cs b/src/marlin.core/models/VersionControlledBranch.cs deleted file mode 100644 index 601f527..0000000 --- a/src/marlin.core/models/VersionControlledBranch.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace marlin.core.models; - -public class VersionControlledBranch -{ - public Guid Id { get; set; } - - public Guid SourceId { get; set; } - public VersionControlSource Source { get; set; } - - public Guid CommitId { get; set; } - public VersionControlledCommit Commit { get; set; } - - public string Name { get; set; } -} \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlledCommit.cs b/src/marlin.core/models/VersionControlledCommit.cs deleted file mode 100644 index a7b5e9e..0000000 --- a/src/marlin.core/models/VersionControlledCommit.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace marlin.core.models; - -public class VersionControlledCommit -{ - public Guid Id { get; set; } - - public Guid SourceId { get; set; } - public VersionControlSource Source { get; set; } - public bool BlobsFetched { get; set; } - - public string Hash { get; set; } - - public ICollection Branches { get; set; } - public ICollection Blobs { get; set; } -} \ No newline at end of file diff --git a/src/marlin.core/models/meta/AbstractStep.cs b/src/marlin.core/models/meta/AbstractStep.cs new file mode 100644 index 0000000..7d10643 --- /dev/null +++ b/src/marlin.core/models/meta/AbstractStep.cs @@ -0,0 +1,16 @@ +namespace marlin.core.models.meta; + +public abstract class AbstractStep +{ + protected AbstractStep(string id) + { + this.Id = id; + } + + public string Id { get; } + public string? Name { get; set; } = null; + public string? Run { get; set; } = null; + public Dictionary With { get; set; } = new(); + public FailBehavior? Fail { get; set; } = null; + public abstract IEnumerable GetAncestorsAndThis(Workflow workflow); +} diff --git a/src/marlin.core/models/meta/Artifact.cs b/src/marlin.core/models/meta/Artifact.cs new file mode 100644 index 0000000..dce25a6 --- /dev/null +++ b/src/marlin.core/models/meta/Artifact.cs @@ -0,0 +1,26 @@ +namespace marlin.core.models.meta; + +public class Artifact +{ + public Artifact(string id) + { + this.Id = id; + } + + public string Id { get; } + public string Path { get; set; } + public string Type { get; set; } + public ArtifactHandle Handle { get; set; } = ArtifactHandle.Defer; + + public bool RequiresOrchestratorUpload() + { + return this.Type?.StartsWith("test/") == true || this.Type?.StartsWith("logs/") == true; + } +} + +public enum ArtifactHandle +{ + Immediate, + Step, + Defer +} \ No newline at end of file diff --git a/src/marlin.core/models/meta/ArtifactHandler.cs b/src/marlin.core/models/meta/ArtifactHandler.cs new file mode 100644 index 0000000..51b6519 --- /dev/null +++ b/src/marlin.core/models/meta/ArtifactHandler.cs @@ -0,0 +1,12 @@ +namespace marlin.core.models.meta; + +public class ArtifactHandler +{ + public ArtifactHandler(string id) + { + this.Id = id; + } + + public string Id { get; } + public ArtifactHandlerStep? Handle { get; set; } +} diff --git a/src/marlin.core/models/meta/ArtifactHandlerStep.cs b/src/marlin.core/models/meta/ArtifactHandlerStep.cs new file mode 100644 index 0000000..bfcc9b9 --- /dev/null +++ b/src/marlin.core/models/meta/ArtifactHandlerStep.cs @@ -0,0 +1,14 @@ +namespace marlin.core.models.meta; + +public class ArtifactHandlerStep : AbstractStep +{ + public ArtifactHandlerStep(string id) : base(id) + { + } + + public CleanupStep? Cleanup { get; set; } = null; + public override IEnumerable GetAncestorsAndThis(Workflow workflow) + { + yield return this; + } +} diff --git a/src/marlin.core/models/meta/CleanupAfter.cs b/src/marlin.core/models/meta/CleanupAfter.cs new file mode 100644 index 0000000..0ca8f5a --- /dev/null +++ b/src/marlin.core/models/meta/CleanupAfter.cs @@ -0,0 +1,7 @@ +namespace marlin.core.models.meta; + +public enum CleanupAfter +{ + Step, + Recipe +} \ No newline at end of file diff --git a/src/marlin.core/models/meta/CleanupStep.cs b/src/marlin.core/models/meta/CleanupStep.cs new file mode 100644 index 0000000..b2d895b --- /dev/null +++ b/src/marlin.core/models/meta/CleanupStep.cs @@ -0,0 +1,14 @@ +namespace marlin.core.models.meta; + +public class CleanupStep : AbstractStep +{ + public CleanupStep(string id) : base(id) + { + } + + public CleanupAfter After { get; set; } + public override IEnumerable GetAncestorsAndThis(Workflow workflow) + { + yield return this; + } +} \ No newline at end of file diff --git a/src/marlin.core/models/meta/FailBehavior.cs b/src/marlin.core/models/meta/FailBehavior.cs new file mode 100644 index 0000000..6084f5d --- /dev/null +++ b/src/marlin.core/models/meta/FailBehavior.cs @@ -0,0 +1,8 @@ +namespace marlin.core.models.meta; + +public enum FailBehavior +{ + Continue, + Stop, + StopAfter +} \ No newline at end of file diff --git a/src/marlin.core/models/meta/Recipe.cs b/src/marlin.core/models/meta/Recipe.cs new file mode 100644 index 0000000..33ed814 --- /dev/null +++ b/src/marlin.core/models/meta/Recipe.cs @@ -0,0 +1,36 @@ +namespace marlin.core.models.meta; + +public class Recipe +{ + public Recipe(string id) + { + this.Id = id; + } + + public string Id { get; } + public string Name { get; set; } + + public string? RunsOnId { get; set; } + public List RequireIds { get; set; } = []; + public Dictionary Steps { get; set; } = new(); + public Dictionary Parameters { get; set; } = new(); + public Dictionary Environment { get; set; } = new(); + + public RequirementSet? GetRunsOn(Workflow workflow) => + this.RunsOnId != null ? workflow.Requirements[this.RunsOnId] : null; + + public IEnumerable GetRequires(Workflow workflow) => + this.RequireIds.Select(id => workflow.Recipes[id]); + + public Dictionary CreateArguments() + { + Dictionary args = new(); + foreach (var parameter in this.Parameters) + args.Add(parameter.Key, parameter.Value); + + foreach (var environment in this.Environment) + args.Add($"env.{environment.Key}", environment.Value); + + return args; + } +} diff --git a/src/marlin.core/models/meta/RecipeStep.cs b/src/marlin.core/models/meta/RecipeStep.cs new file mode 100644 index 0000000..afe77cf --- /dev/null +++ b/src/marlin.core/models/meta/RecipeStep.cs @@ -0,0 +1,28 @@ +namespace marlin.core.models.meta; + +public class RecipeStep : AbstractStep +{ + public RecipeStep(string id) : base(id) + { + } + + public string? BaseId { get; set; } + public CleanupStep? Cleanup { get; set; } + public Dictionary Artifacts { get; set; } = new(); + public List Ingest { get; set; } = new(); + + public RecipeStep? GetBase(Workflow workflow) => + this.BaseId != null ? workflow.Steps[this.BaseId] : null; + + public override IEnumerable GetAncestorsAndThis(Workflow workflow) + { + yield return this; + + var step = this.GetBase(workflow); + while (step != null) + { + yield return step; + step = step.GetBase(workflow); + } + } +} diff --git a/src/marlin.core/models/meta/RequirementSet.cs b/src/marlin.core/models/meta/RequirementSet.cs new file mode 100644 index 0000000..13a3daa --- /dev/null +++ b/src/marlin.core/models/meta/RequirementSet.cs @@ -0,0 +1,13 @@ +namespace marlin.core.models.meta; + +public class RequirementSet +{ + public RequirementSet(string id) + { + this.Id = id; + } + + public string Id { get; } + public string? Base { get; set; } + public Dictionary Has { get; set; } = new(); +} diff --git a/src/marlin.core/models/meta/Workflow.cs b/src/marlin.core/models/meta/Workflow.cs new file mode 100644 index 0000000..d7a200a --- /dev/null +++ b/src/marlin.core/models/meta/Workflow.cs @@ -0,0 +1,9 @@ +namespace marlin.core.models.meta; + +public class Workflow +{ + public Dictionary Recipes { get; set; } = new(); + public Dictionary Steps { get; set; } = new(); + public Dictionary ArtifactHandlers { get; set; } = new(); + public Dictionary Requirements { get; set; } = new(); +} diff --git a/src/marlin.core/models/meta/dto/ArtifactDto.cs b/src/marlin.core/models/meta/dto/ArtifactDto.cs new file mode 100644 index 0000000..aa2db76 --- /dev/null +++ b/src/marlin.core/models/meta/dto/ArtifactDto.cs @@ -0,0 +1,8 @@ +namespace marlin.core.models.meta.dto; + +public sealed record ArtifactDto +{ + public string? Path { get; init; } + public string? As { get; init; } + public ArtifactHandle? Handle { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs b/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs new file mode 100644 index 0000000..0e51a52 --- /dev/null +++ b/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs @@ -0,0 +1,6 @@ +namespace marlin.core.models.meta.dto; + +public sealed record ArtifactHandlerDto +{ + public StepDtoBase? Handle { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/CleanupStepDto.cs b/src/marlin.core/models/meta/dto/CleanupStepDto.cs new file mode 100644 index 0000000..606619b --- /dev/null +++ b/src/marlin.core/models/meta/dto/CleanupStepDto.cs @@ -0,0 +1,6 @@ +namespace marlin.core.models.meta.dto; + +public sealed record CleanupStepDto : StepDtoBase +{ + public CleanupAfter? After { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/RecipeDto.cs b/src/marlin.core/models/meta/dto/RecipeDto.cs new file mode 100644 index 0000000..5e311b9 --- /dev/null +++ b/src/marlin.core/models/meta/dto/RecipeDto.cs @@ -0,0 +1,11 @@ +namespace marlin.core.models.meta.dto; + +public sealed record RecipeDto +{ + public string? Name { get; init; } + public string? RunsOn { get; init; } + public List? Requires { get; init; } + public Dictionary? Steps { get; init; } + public Dictionary? Parameters { get; init; } + public Dictionary? Environment { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/RequirementSetDto.cs b/src/marlin.core/models/meta/dto/RequirementSetDto.cs new file mode 100644 index 0000000..165ed66 --- /dev/null +++ b/src/marlin.core/models/meta/dto/RequirementSetDto.cs @@ -0,0 +1,7 @@ +namespace marlin.core.models.meta.dto; + +public sealed record RequirementSetDto +{ + public string? Base { get; init; } + public Dictionary? Has { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/StepDto.cs b/src/marlin.core/models/meta/dto/StepDto.cs new file mode 100644 index 0000000..57e4092 --- /dev/null +++ b/src/marlin.core/models/meta/dto/StepDto.cs @@ -0,0 +1,9 @@ +namespace marlin.core.models.meta.dto; + +public sealed record StepDto : StepDtoBase +{ + public string? Base { get; init; } + public CleanupStepDto? Cleanup { get; init; } + public Dictionary? Artifacts { get; init; } + public List? Ingest { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/StepDtoBase.cs b/src/marlin.core/models/meta/dto/StepDtoBase.cs new file mode 100644 index 0000000..6f4e36e --- /dev/null +++ b/src/marlin.core/models/meta/dto/StepDtoBase.cs @@ -0,0 +1,9 @@ +namespace marlin.core.models.meta.dto; + +public record StepDtoBase +{ + public string? Name { get; init; } + public string? Run { get; init; } + public Dictionary? With { get; init; } + public FailBehavior? Fail { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/WorkflowFileDto.cs b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs new file mode 100644 index 0000000..10997d6 --- /dev/null +++ b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs @@ -0,0 +1,9 @@ +namespace marlin.core.models.meta.dto; + +public sealed record WorkflowFileDto +{ + public Dictionary? Recipes { get; init; } + public Dictionary? Steps { get; init; } + public Dictionary? Requirements { get; init; } + public Dictionary? ArtifactHandlers { get; init; } +} diff --git a/src/marlin.core/models/plan/CommandResult.cs b/src/marlin.core/models/plan/CommandResult.cs new file mode 100644 index 0000000..3385e84 --- /dev/null +++ b/src/marlin.core/models/plan/CommandResult.cs @@ -0,0 +1,17 @@ +namespace marlin.core.models.plan; + +[Flags] +public enum CommandResult : byte +{ + Success = 1, + Failure = 2, + Unconditional = Success | Failure, +} + +public static class CommandResultExtensions +{ + public static bool Matches(this CommandResult self, CommandResult contains) + { + return (self & contains) != 0; + } +} \ No newline at end of file diff --git a/src/marlin.core/models/plan/CommandResultSelector.cs b/src/marlin.core/models/plan/CommandResultSelector.cs new file mode 100644 index 0000000..92c41c7 --- /dev/null +++ b/src/marlin.core/models/plan/CommandResultSelector.cs @@ -0,0 +1,13 @@ +namespace marlin.core.models.plan; + +public class CommandResultSelector +{ + public ExecutionCommand Command { get; } + public CommandResult Result { get; } + + public CommandResultSelector(ExecutionCommand command, CommandResult result) + { + this.Command = command; + this.Result = result; + } +} \ No newline at end of file diff --git a/src/marlin.core/models/plan/CommandRole.cs b/src/marlin.core/models/plan/CommandRole.cs new file mode 100644 index 0000000..8005532 --- /dev/null +++ b/src/marlin.core/models/plan/CommandRole.cs @@ -0,0 +1,8 @@ +namespace marlin.core.models.plan; + +public enum CommandRole +{ + Step = 0, + ArtifactHandler = 1, + Cleanup = 2 +} \ No newline at end of file diff --git a/src/marlin.core/models/plan/ExecutionCommand.cs b/src/marlin.core/models/plan/ExecutionCommand.cs new file mode 100644 index 0000000..124da63 --- /dev/null +++ b/src/marlin.core/models/plan/ExecutionCommand.cs @@ -0,0 +1,15 @@ +namespace marlin.core.models.plan; + +using meta; + +public class ExecutionCommand +{ + public required string Id; + public required string Name; + public required string Command; + public required CommandRole Role; + public required bool Deferred; + public CommandResultSelector? ResultSelector; + + public required FailBehavior FailBehavior; +} \ No newline at end of file diff --git a/src/marlin.core/models/plan/ExecutionPlan.cs b/src/marlin.core/models/plan/ExecutionPlan.cs new file mode 100644 index 0000000..24c40e4 --- /dev/null +++ b/src/marlin.core/models/plan/ExecutionPlan.cs @@ -0,0 +1,174 @@ +namespace marlin.core.models.plan; + +using meta; +using util; + +public class ExecutionPlan +{ + public IEnumerable Components => [..this._commands, ..this._deferredArtifactHandler, ..this._deferredCleanup]; + + private readonly List _commands; + private readonly List _deferredArtifactHandler; + private readonly List _deferredCleanup; + + public ExecutionPlan() + { + this._commands = new(); + this._deferredArtifactHandler = new(); + this._deferredCleanup = new(); + } + + private void AddCommand(ExecutionCommand command) + { + if (!command.Deferred) + this._commands.Add(command); + + else + { + switch (command.Role) + { + case CommandRole.Cleanup: + this._deferredCleanup.Add(command); + break; + + case CommandRole.ArtifactHandler: + this._deferredArtifactHandler.Add(command); + break; + + default: + throw new InvalidOperationException($"Cannot defer {command.Role}"); + } + } + } + + public static ExecutionPlan Create(Workflow workflow, Recipe recipe) + { + ExecutionPlan plan = new(); + + AddToExecutionPlan(plan, workflow, recipe, null); + + return plan; + } + + private static ExecutionCommand? AddToExecutionPlan(ExecutionPlan plan, Workflow workflow, Recipe recipe, ExecutionCommand? previousCommand) + { + foreach (var dependentRecipe in recipe.GetRequires(workflow)) + previousCommand = AddToExecutionPlan(plan, workflow, dependentRecipe, previousCommand); + + Dictionary recipeArgs = recipe.CreateArguments(); + + foreach (var step in recipe.Steps.Values) + { + var command = CreateCommand(step, recipe, workflow, false, recipeArgs, out var commandArgs); + if (previousCommand != null) + { + command.ResultSelector = new CommandResultSelector(previousCommand, + previousCommand.FailBehavior switch + { + FailBehavior.Continue => CommandResult.Unconditional, + FailBehavior.Stop => CommandResult.Success, + FailBehavior.StopAfter => CommandResult.Success, + + _ => throw new ArgumentOutOfRangeException() + }); + } + + plan.AddCommand(command); + + var cleanupCommand = CreateCleanupCommand(step, command, recipe, workflow, commandArgs); + if (cleanupCommand != null) + plan.AddCommand(cleanupCommand); + + previousCommand = command; + } + + return previousCommand; + } + + private static ExecutionCommand CreateCommand( + AbstractStep step, + Recipe recipe, + Workflow workflow, + bool defer, + Dictionary baseArgs, + out Dictionary createdArgs) + { + string? name = null; + string? run = null; + FailBehavior? fail = null; + + Dictionary args = new(baseArgs); + + foreach (var stepHierarchy in step.GetAncestorsAndThis(workflow)) + { + name ??= stepHierarchy.Name; + run ??= stepHierarchy.Run; + fail ??= stepHierarchy.Fail; + + foreach (var kvp in stepHierarchy.With) + args.TryAdd(kvp.Key, kvp.Value); + } + + name ??= step.Id; + fail ??= FailBehavior.Stop; + + if (run == null) + throw new Exception($"No run found for {recipe.Name}.{name}"); + + run = VariableExpander.Expand(run, args); + + ExecutionCommand stepCommand = new ExecutionCommand + { + Id = step.Id, + Name = name, + Command = run, + Role = step switch + { + ArtifactHandlerStep => CommandRole.ArtifactHandler, + RecipeStep => CommandRole.Step, + CleanupStep => CommandRole.Cleanup, + _ => throw new ArgumentOutOfRangeException() + }, + FailBehavior = fail.Value, + Deferred = defer + }; + + createdArgs = args; + return stepCommand; + } + + private static ExecutionCommand? CreateCleanupCommand(RecipeStep parentStep, + ExecutionCommand parentCommand, + Recipe recipe, + Workflow workflow, + Dictionary baseArgs) + { + var cleanupStep = GetCleanup(parentStep, workflow); + if (cleanupStep == null) + return null; + + var cleanupCommand = CreateCommand(cleanupStep, recipe, workflow, + cleanupStep.After == CleanupAfter.Recipe, baseArgs, out _); + + cleanupCommand.ResultSelector = new CommandResultSelector(parentCommand, + parentCommand.FailBehavior switch + { + FailBehavior.Stop => CommandResult.Success, + FailBehavior.StopAfter => CommandResult.Unconditional, + FailBehavior.Continue => CommandResult.Unconditional, + + _ => throw new ArgumentOutOfRangeException() + }); + + return cleanupCommand; + } + + private static CleanupStep? GetCleanup(RecipeStep step, Workflow workflow) + { + return step + .GetAncestorsAndThis(workflow) + .OfType() + .Select(x => x.Cleanup) + .FirstOrDefault(x => x != null); + } +} diff --git a/src/marlin.core/util/SecureTokenGenerator.cs b/src/marlin.core/util/SecureTokenGenerator.cs new file mode 100644 index 0000000..f99918c --- /dev/null +++ b/src/marlin.core/util/SecureTokenGenerator.cs @@ -0,0 +1,16 @@ +namespace marlin.core.util; + +using System.Security.Cryptography; + +public class SecureTokenGenerator +{ + private const string ALLOWED_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + private static string Create(int length = 32) + { + return RandomNumberGenerator.GetString(ALLOWED_CHARS, length); + } + + public static string CreateAgentToken() => "ag-" + Create(); + public static string CreateClaimToken() => "cl-" + Create(); +} \ No newline at end of file diff --git a/src/marlin.core/util/VariableExpander.cs b/src/marlin.core/util/VariableExpander.cs new file mode 100644 index 0000000..3c3636e --- /dev/null +++ b/src/marlin.core/util/VariableExpander.cs @@ -0,0 +1,140 @@ +namespace marlin.core.util; + +using System.Text; +using System.Text.RegularExpressions; + +public class VariableExpander +{ + public static string Expand(string format, Dictionary variables) + { + StringBuilder output = new(); + + for (int i = 0; i < format.Length; i++) + { + char c = format[i]; + if (c == '{') + { + if (i + 1 >= format.Length) + throw new UnmatchedBraceException("Trailing expression '{'", i, format[i..]); + + if (format[i + 1] == '{') + { + output.Append('{'); + i++; + } + else + { + string expr = GetExpression(format, ref i); + string value = EvaluateExpression(expr, variables); + output.Append(value); + } + } + else if (c == '}') + { + if (i + 1 >= format.Length) + throw new UnmatchedBraceException("Uninitiated expression at '}'", i, format[i..]); + + if (format[i + 1] == '}') + { + output.Append('}'); + i++; + } + else + { + throw new UnmatchedBraceException("Uninitiated expression at '}'", i, format[i..]); + } + } + else + output.Append(c); + } + + string result = output.ToString(); + + // Clean up extra spaces: collapse multiple consecutive spaces (but preserve other whitespace) + result = Regex.Replace(result, @" {2,}", " "); + result = result.Trim(); + + return result; + } + + private static string GetExpression(string format, ref int pos) + { + StringBuilder part = new(); + + if (format[pos] != '{') + throw new InvalidExpressionException(format[pos..], "Expected '{' at start of expression"); + + pos++; + + for (; pos < format.Length; pos++) + { + if (format[pos] == '}') + break; + + part.Append(format[pos]); + } + + if (pos >= format.Length || format[pos] != '}') + throw new UnmatchedBraceException($"Trailing expression {format[pos..]}", pos, format[pos..]); + + return part.ToString(); + } + + private static string EvaluateExpression(string expr, Dictionary variables) + { + Match match; + + const string identifier = "[a-zA-Z0-9-_.]+"; + + // Variable value + match = Regex.Match(expr, $"^{identifier}$"); + if (match.Success) + return GetVariableOrThrow(expr, variables); + + // Conditional argument: only included if variable is set. + // --hello var + match = Regex.Match(expr, $"^(--?{identifier})\\s+({identifier})$"); + if (match.Success) + { + if (variables.TryGetValue(match.Groups[2].Value, out var value)) + return $"{match.Groups[1].Value} {value}"; + + else + return ""; + } + + // Default value, if value is unset, uses default + match = Regex.Match(expr, $@"^({identifier})\s*=\s*(.*)$"); + if (match.Success) + { + if (variables.TryGetValue(match.Groups[1].Value, out var value)) + return value; + else + return match.Groups[2].Value; + } + + // Use value if set + match = Regex.Match(expr, $@"^({identifier})\?\s*(.*)$"); + if (match.Success) + { + if (variables.TryGetValue(match.Groups[1].Value, out var value)) + { + if (string.IsNullOrEmpty(value) || value == "null" || value == "false") + return ""; + + return match.Groups[2].Value; + } + else + return ""; + } + + throw new InvalidExpressionException(expr); + } + + private static string GetVariableOrThrow(string variable, Dictionary variables) + { + return variables.TryGetValue(variable, out var value) + ? value + : throw new VariableNotFoundException(variable); + } +} \ No newline at end of file diff --git a/src/marlin.core/util/VariableExpansionException.cs b/src/marlin.core/util/VariableExpansionException.cs new file mode 100644 index 0000000..3053880 --- /dev/null +++ b/src/marlin.core/util/VariableExpansionException.cs @@ -0,0 +1,56 @@ +namespace marlin.core.util; + +/// +/// Base exception for all variable expansion errors +/// +public class VariableExpansionException(string message) : Exception(message); + +/// +/// Thrown when a variable is referenced but not found in the variables dictionary +/// +public class VariableNotFoundException : VariableExpansionException +{ + public string VariableName { get; } + + public VariableNotFoundException(string variableName) + : base($"Variable '{variableName}' not found") + { + this.VariableName = variableName; + } +} + +/// +/// Thrown when an expression has invalid syntax or cannot be parsed +/// +public class InvalidExpressionException : VariableExpansionException +{ + public string Expression { get; } + + public InvalidExpressionException(string expression) + : base($"Invalid expression '{expression}'") + { + this.Expression = expression; + } + + public InvalidExpressionException(string expression, string reason) + : base($"Invalid expression '{expression}': {reason}") + { + this.Expression = expression; + } +} + +/// +/// Thrown when braces are not properly matched or escaped +/// +public class UnmatchedBraceException : VariableExpansionException +{ + public int Position { get; } + public string Remainder { get; } + + public UnmatchedBraceException(string message, int position, string remainder) + : base(message) + { + this.Position = position; + this.Remainder = remainder; + } +} diff --git a/src/marlin.web/Controllers/AgentsController.cs b/src/marlin.web/Controllers/AgentsController.cs new file mode 100644 index 0000000..199acf9 --- /dev/null +++ b/src/marlin.web/Controllers/AgentsController.cs @@ -0,0 +1,89 @@ +namespace marlin.web.Controllers; + +using core.models; +using core.util; +using Data; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Models; + +[Route("agents")] +public class AgentsController(ApplicationDbContext dbContext) : Controller +{ + [HttpGet("list")] + public async Task List(CancellationToken ct) + { + var items = await dbContext + .Agents + .Include(x => x.Jobs.Where(j => j.Status == JobStatus.Active)) + .ToListAsync(ct); + + return this.View(items); + } + + [HttpGet("details/{id}")] + public async Task Details(Guid id, CancellationToken ct) + { + var item = await dbContext.Agents + .Include(x => x.Jobs.Where(j => j.Status == JobStatus.Active)) + .FirstOrDefaultAsync(x => x.Id == id, ct); + + if (item == null) + return this.NotFound(); + + return this.View(item); + } + + [HttpPost("edit/{id}")] + public async Task Edit(Guid id, Agent agent, CancellationToken ct) + { + var item = await dbContext.Agents + .FirstOrDefaultAsync(x => x.Id == id, ct); + + if (item == null) + return this.NotFound(); + + item.Name = agent.Name; + item.MaxConcurrentJobs = agent.MaxConcurrentJobs; + + await dbContext.SaveChangesAsync(ct); + + return this.RedirectToAction("Details", new { id }); + } + + [HttpPost("details/{id}/regenerate-token")] + public async Task RegenerateToken(Guid id, CancellationToken ct) + { + var item = await dbContext.Agents + .FirstOrDefaultAsync(x => x.Id == id, ct); + + if (item == null) + return this.NotFound(); + + item.AuthToken = SecureTokenGenerator.CreateAgentToken(); + + await dbContext.SaveChangesAsync(ct); + + return this.RedirectToAction("Details", new { id }); + } + + [HttpGet("create")] + public IActionResult Create() + { + return this.View(); + } + + [HttpPost("create")] + public async Task CreatePost(Agent agent, CancellationToken ct) + { + agent.Id = Guid.Empty; + agent.AuthToken = SecureTokenGenerator.CreateAgentToken(); + agent.LastSeen = DateTime.UtcNow; + agent.Capabilities = new CapabilitySet(); + + await dbContext.Agents.AddAsync(agent, ct); + await dbContext.SaveChangesAsync(ct); + + return this.RedirectToAction("Details", new { id = agent.Id }); + } +} \ No newline at end of file diff --git a/src/marlin.web/Controllers/DispatchController.cs b/src/marlin.web/Controllers/DispatchController.cs index 6b2285c..a37d5fa 100644 --- a/src/marlin.web/Controllers/DispatchController.cs +++ b/src/marlin.web/Controllers/DispatchController.cs @@ -1,10 +1,71 @@ namespace marlin.web.Controllers; +using core.models; +using core.models.meta; +using core.util; +using Data; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Models; +using Services; +using Util; [Route("api/dispatch")] [ApiController] -public class DispatchController : ControllerBase +[Authorize(Policy = "Agent")] +public class DispatchController(ApplicationDbContext dbContext) : ControllerBase { + [HttpPost("handshake")] + public async Task> Handshake(AgentStatus status, CancellationToken ct) + { + var agentId = User.RequireAgentId(); + + var agent = await dbContext.Agents + .FirstAsync(x => x.Id == agentId, ct); + + foreach (var capability in status.CapabilitySet.Items) + agent.Capabilities.Items[capability.Key] = capability.Value; + + await dbContext.SaveChangesAsync(ct); + + return new AgentConfiguration( + Name: agent.Name, + MaxConcurrentJobs: agent.MaxConcurrentJobs + ); + } -} \ No newline at end of file + [HttpGet("queue")] + public async Task> Queue(CancellationToken ct) + { + var agentId = User.RequireAgentId(); + + var nextJob = await dbContext + .Jobs + .Include(x => x.Source) + .Include(x => x.Branch) + .Include(x => x.Commit) + .Where(x => x.Status == JobStatus.Pending && x.AgentId == null) + .OrderBy(x => x.CreatedAt) + .FirstOrDefaultAsync(ct); + + if (nextJob == null) + return (DispatchJob?) null; + + string claimToken = SecureTokenGenerator.CreateClaimToken(); + nextJob.ClaimToken = claimToken; + nextJob.AgentId = agentId; + + await dbContext.SaveChangesAsync(ct); + + return new DispatchJob( + Id: nextJob.Id, + Token: claimToken, + Vcs: nextJob.Source.UpstreamUrl, + Ref: nextJob.Branch.Name, + Commit: nextJob.Commit.Hash, + Recipe: nextJob.Recipe + ); + } +} + diff --git a/src/marlin.web/Controllers/JobsController.cs b/src/marlin.web/Controllers/JobsController.cs index c86cb2d..552b56c 100644 --- a/src/marlin.web/Controllers/JobsController.cs +++ b/src/marlin.web/Controllers/JobsController.cs @@ -4,25 +4,29 @@ using core.models; using Data; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using Models; using Util; [Route("jobs")] public class JobsController(ApplicationDbContext dbContext) : Controller { - // list jobs (pending, active, cancelled, success, failed), filter to vcs, filter to recipe - public record JobListFilter(JobStatus? Status = null, Guid? Vcs = null, string? Ref = null, string? Recipe = null); - + public record JobListFilter(JobStatus? Status = null, Guid? Vcs = null, Guid? Branch = null, string? Recipe = null, Guid? Agent = null); + [HttpGet("list")] public async Task List([FromQuery] JobListFilter filter, CancellationToken ct) { - var results = await dbContext.Jobs + var results = await dbContext + .Jobs .WhereIf(filter.Status != null, x => x.Status == filter.Status) .WhereIf(filter.Vcs != null, x => x.SourceId == filter.Vcs) - .WhereIf(filter.Ref != null, x => x.Ref == filter.Ref) + .WhereIf(filter.Branch != null, x => x.BranchId == filter.Branch) .WhereIf(filter.Recipe != null, x => x.Recipe == filter.Recipe) + .WhereIf(filter.Agent != null, x => x.AgentId == filter.Agent) .OrderByDescending(x => x.CreatedAt) - + + .Include(x => x.Commit) + .Include(x => x.Branch) .Include(x => x.Source) .Include(x => x.Agent) .ToListAsync(ct); @@ -40,25 +44,34 @@ public class JobsController(ApplicationDbContext dbContext) : Controller [HttpGet("spawn")] public async Task Spawn(CancellationToken ct) { - var sources = await dbContext.Sources + var sources = await dbContext + .Sources .Include(x => x.Branches) .ThenInclude(x => x.Commit) + .ThenInclude(x => x.WorkflowSnapshot) .ToListAsync(ct); var details = new JobSpawnDetails(sources); return this.View(details); } - public record JobSpawnRequest(Guid Vcs, string Ref, string Commit, string Recipe); + public record JobSpawnRequest(Guid Vcs, Guid Branch, string Recipe); [HttpPost("spawn")] public async Task SpawnPost(JobSpawnRequest request, CancellationToken ct) { + var branch = await dbContext + .SourceBranches + .FirstOrDefaultAsync(x => x.Id == request.Branch, ct); + + if (branch == null) + return this.NotFound(); + Job job = new() { - SourceId = request.Vcs, - Ref = request.Ref, - Commit = request.Commit, + SourceId = branch.SourceId, + BranchId = branch.Id, + CommitId = branch.CommitId, Recipe = request.Recipe, CreatedAt = DateTime.UtcNow, diff --git a/src/marlin.web/Controllers/VCSController.cs b/src/marlin.web/Controllers/VCSController.cs index 7370e0d..8086e99 100644 --- a/src/marlin.web/Controllers/VCSController.cs +++ b/src/marlin.web/Controllers/VCSController.cs @@ -27,15 +27,16 @@ public class VCSController(ApplicationDbContext dbContext, VersionControlSyncSer [HttpGet("details/{id}")] public async Task Details(Guid id, CancellationToken ct) { - var item = await dbContext.Sources + var item = await dbContext + .Sources .Include(x => x.Branches) .ThenInclude(x => x.Commit) + .ThenInclude(x => x.WorkflowSnapshot) .FirstOrDefaultAsync(x => x.Id == id, ct); if (item == null) return this.NotFound(); - // Only Id/Label projected — a secret's Content never needs to leave the secrets pages. var secretOptions = await dbContext.Secrets .Where(x => x.SourceId == id) .Select(x => new SecretOption(x.Id, x.Label)) @@ -44,6 +45,37 @@ public class VCSController(ApplicationDbContext dbContext, VersionControlSyncSer return this.View((item, secretOptions)); } + [HttpGet("details/{sourceId}/{commitId}/blobs")] + public async Task Blobs(Guid sourceId, Guid commitId) + { + var blobs = await dbContext + .SourceBlobs + .Include(x => x.Commit) + .ThenInclude(x => x.Source) + .Where(x => x.CommitId == commitId && x.SourceId == sourceId) + .ToListAsync(); + + if (!blobs.Any()) + return this.NotFound(); + + return this.View(blobs); + } + + [HttpGet("details/{sourceId}/{commitId}/workflow")] + public async Task Workflow(Guid sourceId, Guid commitId) + { + var snapshot = await dbContext + .WorkflowSnapshots + .Include(x => x.Commit) + .ThenInclude(x => x.Source) + .FirstOrDefaultAsync(x => x.CommitId == commitId && x.Commit.SourceId == sourceId); + + if (snapshot == null) + return this.NotFound(); + + return this.View(snapshot); + } + [HttpPost("edit/{id}")] public async Task Edit(Guid id, VersionControlSource vcs, CancellationToken ct) { diff --git a/src/marlin.web/Data/ApplicationDbContext.cs b/src/marlin.web/Data/ApplicationDbContext.cs index 2e05835..cb6b05b 100644 --- a/src/marlin.web/Data/ApplicationDbContext.cs +++ b/src/marlin.web/Data/ApplicationDbContext.cs @@ -3,6 +3,7 @@ namespace marlin.web.Data; using System.Linq.Expressions; using System.Text.Json; using core; +using core.functional; using core.models; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; @@ -26,6 +27,7 @@ public class ApplicationDbContext : DbContext public DbSet Secrets => this.Set(); public DbSet Jobs => this.Set(); public DbSet Agents => this.Set(); + public DbSet WorkflowSnapshots => this.Set(); public ApplicationDbContext(DbContextOptions options) : base(options) { } @@ -35,9 +37,7 @@ public class ApplicationDbContext : DbContext modelBuilder .Entity() .Property(p => p.Capabilities) - .HasConversion( - v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null), - v => JsonSerializer.Deserialize(v, (JsonSerializerOptions?)null)!); + .HasJsonConversion(); modelBuilder .Entity() @@ -46,10 +46,19 @@ public class ApplicationDbContext : DbContext modelBuilder .Entity() .Property(p => p.Content) - .HasConversion( - v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null), - v => JsonSerializer.Deserialize>(v, (JsonSerializerOptions?)null)!); - + .HasJsonConversion(); + + modelBuilder + .Entity() + .HasOne(x => x.WorkflowSnapshot) + .WithOne(x => x.Commit) + .HasForeignKey(x => x.CommitId); + + modelBuilder + .Entity() + .Property(p => p.Workflow) + .HasJsonConversion(); + base.OnModelCreating(modelBuilder); } } \ No newline at end of file diff --git a/src/marlin.web/Data/JsonValueComparer.cs b/src/marlin.web/Data/JsonValueComparer.cs new file mode 100644 index 0000000..f07c133 --- /dev/null +++ b/src/marlin.web/Data/JsonValueComparer.cs @@ -0,0 +1,23 @@ +namespace marlin.web.Data; + +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +public static class JsonValueComparer +{ + public static ValueComparer Create() => new( + (a, b) => JsonSerializer.Serialize(a, (JsonSerializerOptions?)null) == JsonSerializer.Serialize(b, (JsonSerializerOptions?)null), + v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null).GetHashCode(), + v => JsonSerializer.Deserialize(JsonSerializer.Serialize(v, (JsonSerializerOptions?)null), (JsonSerializerOptions?)null)!); +} + +public static class PropertyBuilderExtensions +{ + public static PropertyBuilder HasJsonConversion(this PropertyBuilder builder) => + builder.HasConversion( + v => JsonSerializer.Serialize(v, (JsonSerializerOptions?)null), + v => JsonSerializer.Deserialize(v, (JsonSerializerOptions?)null)!, + JsonValueComparer.Create()); +} diff --git a/src/marlin.web/Migrations/20260906003038_AddWorkflowSnapshot.Designer.cs b/src/marlin.web/Migrations/20260906003038_AddWorkflowSnapshot.Designer.cs new file mode 100644 index 0000000..e50bdbd --- /dev/null +++ b/src/marlin.web/Migrations/20260906003038_AddWorkflowSnapshot.Designer.cs @@ -0,0 +1,478 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using marlin.web.Data; + +#nullable disable + +namespace marlin.web.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20260906003038_AddWorkflowSnapshot")] + partial class AddWorkflowSnapshot + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("marlin.core.models.Agent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property("Commit") + .IsRequired() + .HasColumnType("text") + .HasColumnName("commit"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property("Ref") + .IsRequired() + .HasColumnType("text") + .HasColumnName("ref"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_update"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_update_job_id"); + + b.ToTable("job_status_update", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccessSecretId") + .HasColumnType("uuid") + .HasColumnName("access_secret_id"); + + b.Property("BranchFilter") + .IsRequired() + .HasColumnType("text") + .HasColumnName("branch_filter"); + + b.Property("MainBranch") + .IsRequired() + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + + b.Property("UpstreamUrl") + .IsRequired() + .HasColumnType("text") + .HasColumnName("upstream_url"); + + b.HasKey("Id") + .HasName("pk_sources"); + + b.ToTable("sources", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_blobs"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_blobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_blobs_source_id"); + + b.ToTable("source_blobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_branches_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BlobsFetched") + .HasColumnType("boolean") + .HasColumnName("blobs_fetched"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("WorkflowScanned") + .HasColumnType("boolean") + .HasColumnName("workflow_scanned"); + + b.HasKey("Id") + .HasName("pk_source_commits"); + + b.HasAlternateKey("SourceId", "Hash") + .HasName("ak_source_commits_source_id_hash"); + + b.ToTable("source_commits", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.WorkflowSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("ScannedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("scanned_at"); + + b.Property("Workflow") + .IsRequired() + .HasColumnType("text") + .HasColumnName("workflow"); + + b.HasKey("Id") + .HasName("pk_workflow_snapshots"); + + b.HasIndex("CommitId") + .IsUnique() + .HasDatabaseName("ix_workflow_snapshots_commit_id"); + + b.ToTable("workflow_snapshots", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("Kind") + .HasColumnType("text") + .HasColumnName("kind"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("MayReveal") + .HasColumnType("boolean") + .HasColumnName("may_reveal"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_secrets"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_secrets_source_id"); + + b.ToTable("secrets", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.HasOne("marlin.core.models.Agent", "Agent") + .WithMany() + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.HasOne("marlin.core.models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_update_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Blobs") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Branches") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_commits_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.WorkflowSnapshot", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithOne("WorkflowSnapshot") + .HasForeignKey("marlin.core.models.WorkflowSnapshot", "CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_workflow_snapshots_source_commits_commit_id"); + + b.Navigation("Commit"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_secrets_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Navigation("Blobs"); + + b.Navigation("Branches"); + + b.Navigation("WorkflowSnapshot"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/marlin.web/Migrations/20260906003038_AddWorkflowSnapshot.cs b/src/marlin.web/Migrations/20260906003038_AddWorkflowSnapshot.cs new file mode 100644 index 0000000..d47ee9e --- /dev/null +++ b/src/marlin.web/Migrations/20260906003038_AddWorkflowSnapshot.cs @@ -0,0 +1,59 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace marlin.web.Migrations +{ + /// + public partial class AddWorkflowSnapshot : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "workflow_scanned", + table: "source_commits", + type: "boolean", + nullable: false, + defaultValue: false); + + migrationBuilder.CreateTable( + name: "workflow_snapshots", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + commit_id = table.Column(type: "uuid", nullable: false), + scanned_at = table.Column(type: "timestamp with time zone", nullable: false), + workflow = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_workflow_snapshots", x => x.id); + table.ForeignKey( + name: "fk_workflow_snapshots_source_commits_commit_id", + column: x => x.commit_id, + principalTable: "source_commits", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_workflow_snapshots_commit_id", + table: "workflow_snapshots", + column: "commit_id", + unique: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "workflow_snapshots"); + + migrationBuilder.DropColumn( + name: "workflow_scanned", + table: "source_commits"); + } + } +} diff --git a/src/marlin.web/Migrations/20260906082039_AddAgentToken.Designer.cs b/src/marlin.web/Migrations/20260906082039_AddAgentToken.Designer.cs new file mode 100644 index 0000000..c39db32 --- /dev/null +++ b/src/marlin.web/Migrations/20260906082039_AddAgentToken.Designer.cs @@ -0,0 +1,488 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using marlin.web.Data; + +#nullable disable + +namespace marlin.web.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20260906082039_AddAgentToken")] + partial class AddAgentToken + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("marlin.core.models.Agent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AuthToken") + .IsRequired() + .HasColumnType("text") + .HasColumnName("auth_token"); + + b.Property("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property("Commit") + .IsRequired() + .HasColumnType("text") + .HasColumnName("commit"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property("Ref") + .IsRequired() + .HasColumnType("text") + .HasColumnName("ref"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_update"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_update_job_id"); + + b.ToTable("job_status_update", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccessSecretId") + .HasColumnType("uuid") + .HasColumnName("access_secret_id"); + + b.Property("BranchFilter") + .IsRequired() + .HasColumnType("text") + .HasColumnName("branch_filter"); + + b.Property("MainBranch") + .IsRequired() + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + + b.Property("UpstreamUrl") + .IsRequired() + .HasColumnType("text") + .HasColumnName("upstream_url"); + + b.HasKey("Id") + .HasName("pk_sources"); + + b.ToTable("sources", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_blobs"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_blobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_blobs_source_id"); + + b.ToTable("source_blobs", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_branches_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BlobsFetched") + .HasColumnType("boolean") + .HasColumnName("blobs_fetched"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("WorkflowScanned") + .HasColumnType("boolean") + .HasColumnName("workflow_scanned"); + + b.HasKey("Id") + .HasName("pk_source_commits"); + + b.HasAlternateKey("SourceId", "Hash") + .HasName("ak_source_commits_source_id_hash"); + + b.ToTable("source_commits", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.WorkflowSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("ScannedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("scanned_at"); + + b.Property("Workflow") + .IsRequired() + .HasColumnType("text") + .HasColumnName("workflow"); + + b.HasKey("Id") + .HasName("pk_workflow_snapshots"); + + b.HasIndex("CommitId") + .IsUnique() + .HasDatabaseName("ix_workflow_snapshots_commit_id"); + + b.ToTable("workflow_snapshots", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("Kind") + .HasColumnType("text") + .HasColumnName("kind"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("MayReveal") + .HasColumnType("boolean") + .HasColumnName("may_reveal"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_secrets"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_secrets_source_id"); + + b.ToTable("secrets", (string)null); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.HasOne("marlin.core.models.Agent", "Agent") + .WithMany("Jobs") + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + { + b.HasOne("marlin.core.models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_update_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Blobs") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithMany("Branches") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_source_commits_commit_id"); + + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_commits_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.WorkflowSnapshot", b => + { + b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + .WithOne("WorkflowSnapshot") + .HasForeignKey("marlin.core.models.WorkflowSnapshot", "CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_workflow_snapshots_source_commits_commit_id"); + + b.Navigation("Commit"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.HasOne("marlin.core.models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_secrets_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.core.models.Agent", b => + { + b.Navigation("Jobs"); + }); + + modelBuilder.Entity("marlin.core.models.Job", b => + { + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); + + modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + { + b.Navigation("Blobs"); + + b.Navigation("Branches"); + + b.Navigation("WorkflowSnapshot"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/marlin.web/Migrations/20260906082039_AddAgentToken.cs b/src/marlin.web/Migrations/20260906082039_AddAgentToken.cs new file mode 100644 index 0000000..6725186 --- /dev/null +++ b/src/marlin.web/Migrations/20260906082039_AddAgentToken.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace marlin.web.Migrations +{ + /// + public partial class AddAgentToken : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "auth_token", + table: "agents", + type: "text", + nullable: false, + defaultValue: ""); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "auth_token", + table: "agents"); + } + } +} diff --git a/src/marlin.web/Migrations/20260908142748_CommitAlt.Designer.cs b/src/marlin.web/Migrations/20260908142748_CommitAlt.Designer.cs new file mode 100644 index 0000000..5b64d5d --- /dev/null +++ b/src/marlin.web/Migrations/20260908142748_CommitAlt.Designer.cs @@ -0,0 +1,514 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using marlin.web.Data; + +#nullable disable + +namespace marlin.web.Migrations +{ + [DbContext(typeof(ApplicationDbContext))] + [Migration("20260908142748_CommitAlt")] + partial class CommitAlt + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("marlin.web.Models.Agent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AuthToken") + .IsRequired() + .HasColumnType("text") + .HasColumnName("auth_token"); + + b.Property("Capabilities") + .IsRequired() + .HasColumnType("text") + .HasColumnName("capabilities"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("MaxConcurrentJobs") + .HasColumnType("integer") + .HasColumnName("max_concurrent_jobs"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_agents"); + + b.ToTable("agents", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AgentId") + .HasColumnType("uuid") + .HasColumnName("agent_id"); + + b.Property("BranchId") + .HasColumnType("uuid") + .HasColumnName("branch_id"); + + b.Property("ClaimToken") + .HasColumnType("text") + .HasColumnName("claim_token"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedReason") + .IsRequired() + .HasColumnType("text") + .HasColumnName("created_reason"); + + b.Property("Recipe") + .IsRequired() + .HasColumnType("text") + .HasColumnName("recipe"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_jobs"); + + b.HasIndex("AgentId") + .HasDatabaseName("ix_jobs_agent_id"); + + b.HasIndex("BranchId") + .HasDatabaseName("ix_jobs_branch_id"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_jobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_jobs_source_id"); + + b.ToTable("jobs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("At") + .HasColumnType("timestamp with time zone") + .HasColumnName("at"); + + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_job_status_update"); + + b.HasIndex("JobId") + .HasDatabaseName("ix_job_status_update_job_id"); + + b.ToTable("job_status_update", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("Kind") + .HasColumnType("text") + .HasColumnName("kind"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("MayReveal") + .HasColumnType("boolean") + .HasColumnName("may_reveal"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_secrets"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_secrets_source_id"); + + b.ToTable("secrets", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AccessSecretId") + .HasColumnType("uuid") + .HasColumnName("access_secret_id"); + + b.Property("BranchFilter") + .IsRequired() + .HasColumnType("text") + .HasColumnName("branch_filter"); + + b.Property("MainBranch") + .IsRequired() + .HasColumnType("text") + .HasColumnName("main_branch"); + + b.Property("MinPollingInterval") + .HasColumnType("integer") + .HasColumnName("min_polling_interval"); + + b.Property("UpstreamUrl") + .IsRequired() + .HasColumnType("text") + .HasColumnName("upstream_url"); + + b.HasKey("Id") + .HasName("pk_sources"); + + b.ToTable("sources", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_blobs"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_blobs_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_blobs_source_id"); + + b.ToTable("source_blobs", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_source_branches"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_source_branches_commit_id"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_source_branches_source_id"); + + b.ToTable("source_branches", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BlobsFetched") + .HasColumnType("boolean") + .HasColumnName("blobs_fetched"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("text") + .HasColumnName("hash"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.Property("WorkflowScanned") + .HasColumnType("boolean") + .HasColumnName("workflow_scanned"); + + b.HasKey("Id") + .HasName("pk_source_commits"); + + b.HasAlternateKey("SourceId", "Hash") + .HasName("ak_source_commits_source_id_hash"); + + b.ToTable("source_commits", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); + + b.Property("ScannedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("scanned_at"); + + b.Property("Workflow") + .IsRequired() + .HasColumnType("text") + .HasColumnName("workflow"); + + b.HasKey("Id") + .HasName("pk_workflow_snapshots"); + + b.HasIndex("CommitId") + .IsUnique() + .HasDatabaseName("ix_workflow_snapshots_commit_id"); + + b.ToTable("workflow_snapshots", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.HasOne("marlin.web.Models.Agent", "Agent") + .WithMany("Jobs") + .HasForeignKey("AgentId") + .HasConstraintName("fk_jobs_agents_agent_id"); + + b.HasOne("marlin.web.Models.VersionControlledBranch", "Branch") + .WithMany() + .HasForeignKey("BranchId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_branches_branch_id"); + + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany() + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_sources_source_id"); + + b.Navigation("Agent"); + + b.Navigation("Branch"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => + { + b.HasOne("marlin.web.Models.Job", "Job") + .WithMany("StatusUpdates") + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_job_status_update_jobs_job_id"); + + b.Navigation("Job"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_secrets_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany("Blobs") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_blobs_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany("Branches") + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany("Branches") + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_branches_sources_source_id"); + + b.Navigation("Commit"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_source_commits_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithOne("WorkflowSnapshot") + .HasForeignKey("marlin.web.Models.WorkflowSnapshot", "CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_workflow_snapshots_source_commits_commit_id"); + + b.Navigation("Commit"); + }); + + modelBuilder.Entity("marlin.web.Models.Agent", b => + { + b.Navigation("Jobs"); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => + { + b.Navigation("StatusUpdates"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => + { + b.Navigation("Branches"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => + { + b.Navigation("Blobs"); + + b.Navigation("Branches"); + + b.Navigation("WorkflowSnapshot"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/marlin.web/Migrations/20260908142748_CommitAlt.cs b/src/marlin.web/Migrations/20260908142748_CommitAlt.cs new file mode 100644 index 0000000..b607d31 --- /dev/null +++ b/src/marlin.web/Migrations/20260908142748_CommitAlt.cs @@ -0,0 +1,115 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace marlin.web.Migrations +{ + /// + public partial class CommitAlt : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "commit", + table: "jobs"); + + migrationBuilder.DropColumn( + name: "ref", + table: "jobs"); + + migrationBuilder.AddColumn( + name: "branch_id", + table: "jobs", + type: "uuid", + nullable: false, + defaultValue: new Guid("00000000-0000-0000-0000-000000000000")); + + migrationBuilder.AddColumn( + name: "claim_token", + table: "jobs", + type: "text", + nullable: true); + + migrationBuilder.AddColumn( + name: "commit_id", + table: "jobs", + type: "uuid", + nullable: false, + defaultValue: new Guid("00000000-0000-0000-0000-000000000000")); + + migrationBuilder.CreateIndex( + name: "ix_jobs_branch_id", + table: "jobs", + column: "branch_id"); + + migrationBuilder.CreateIndex( + name: "ix_jobs_commit_id", + table: "jobs", + column: "commit_id"); + + migrationBuilder.AddForeignKey( + name: "fk_jobs_source_branches_branch_id", + table: "jobs", + column: "branch_id", + principalTable: "source_branches", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + + migrationBuilder.AddForeignKey( + name: "fk_jobs_source_commits_commit_id", + table: "jobs", + column: "commit_id", + principalTable: "source_commits", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "fk_jobs_source_branches_branch_id", + table: "jobs"); + + migrationBuilder.DropForeignKey( + name: "fk_jobs_source_commits_commit_id", + table: "jobs"); + + migrationBuilder.DropIndex( + name: "ix_jobs_branch_id", + table: "jobs"); + + migrationBuilder.DropIndex( + name: "ix_jobs_commit_id", + table: "jobs"); + + migrationBuilder.DropColumn( + name: "branch_id", + table: "jobs"); + + migrationBuilder.DropColumn( + name: "claim_token", + table: "jobs"); + + migrationBuilder.DropColumn( + name: "commit_id", + table: "jobs"); + + migrationBuilder.AddColumn( + name: "commit", + table: "jobs", + type: "text", + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "ref", + table: "jobs", + type: "text", + nullable: false, + defaultValue: ""); + } + } +} diff --git a/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs b/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs index 8de35de..411706b 100644 --- a/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs +++ b/src/marlin.web/Migrations/ApplicationDbContextModelSnapshot.cs @@ -22,13 +22,18 @@ namespace marlin.web.Migrations NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - modelBuilder.Entity("marlin.core.models.Agent", b => + modelBuilder.Entity("marlin.web.Models.Agent", b => { b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("uuid") .HasColumnName("id"); + b.Property("AuthToken") + .IsRequired() + .HasColumnType("text") + .HasColumnName("auth_token"); + b.Property("Capabilities") .IsRequired() .HasColumnType("text") @@ -53,7 +58,7 @@ namespace marlin.web.Migrations b.ToTable("agents", (string)null); }); - modelBuilder.Entity("marlin.core.models.Job", b => + modelBuilder.Entity("marlin.web.Models.Job", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -64,10 +69,17 @@ namespace marlin.web.Migrations .HasColumnType("uuid") .HasColumnName("agent_id"); - b.Property("Commit") - .IsRequired() + b.Property("BranchId") + .HasColumnType("uuid") + .HasColumnName("branch_id"); + + b.Property("ClaimToken") .HasColumnType("text") - .HasColumnName("commit"); + .HasColumnName("claim_token"); + + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); b.Property("CreatedAt") .HasColumnType("timestamp with time zone") @@ -83,11 +95,6 @@ namespace marlin.web.Migrations .HasColumnType("text") .HasColumnName("recipe"); - b.Property("Ref") - .IsRequired() - .HasColumnType("text") - .HasColumnName("ref"); - b.Property("SourceId") .HasColumnType("uuid") .HasColumnName("source_id"); @@ -102,13 +109,19 @@ namespace marlin.web.Migrations b.HasIndex("AgentId") .HasDatabaseName("ix_jobs_agent_id"); + b.HasIndex("BranchId") + .HasDatabaseName("ix_jobs_branch_id"); + + b.HasIndex("CommitId") + .HasDatabaseName("ix_jobs_commit_id"); + b.HasIndex("SourceId") .HasDatabaseName("ix_jobs_source_id"); b.ToTable("jobs", (string)null); }); - modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -136,7 +149,45 @@ namespace marlin.web.Migrations b.ToTable("job_status_update", (string)null); }); - modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("Kind") + .HasColumnType("text") + .HasColumnName("kind"); + + b.Property("Label") + .IsRequired() + .HasColumnType("text") + .HasColumnName("label"); + + b.Property("MayReveal") + .HasColumnType("boolean") + .HasColumnName("may_reveal"); + + b.Property("SourceId") + .HasColumnType("uuid") + .HasColumnName("source_id"); + + b.HasKey("Id") + .HasName("pk_secrets"); + + b.HasIndex("SourceId") + .HasDatabaseName("ix_secrets_source_id"); + + b.ToTable("secrets", (string)null); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -172,7 +223,7 @@ namespace marlin.web.Migrations b.ToTable("sources", (string)null); }); - modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -209,7 +260,7 @@ namespace marlin.web.Migrations b.ToTable("source_blobs", (string)null); }); - modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -241,7 +292,7 @@ namespace marlin.web.Migrations b.ToTable("source_branches", (string)null); }); - modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -261,6 +312,10 @@ namespace marlin.web.Migrations .HasColumnType("uuid") .HasColumnName("source_id"); + b.Property("WorkflowScanned") + .HasColumnType("boolean") + .HasColumnName("workflow_scanned"); + b.HasKey("Id") .HasName("pk_source_commits"); @@ -270,52 +325,58 @@ namespace marlin.web.Migrations b.ToTable("source_commits", (string)null); }); - modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => { b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("uuid") .HasColumnName("id"); - b.Property("Content") - .IsRequired() - .HasColumnType("text") - .HasColumnName("content"); + b.Property("CommitId") + .HasColumnType("uuid") + .HasColumnName("commit_id"); - b.Property("Kind") - .HasColumnType("text") - .HasColumnName("kind"); + b.Property("ScannedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("scanned_at"); - b.Property("Label") + b.Property("Workflow") .IsRequired() .HasColumnType("text") - .HasColumnName("label"); - - b.Property("MayReveal") - .HasColumnType("boolean") - .HasColumnName("may_reveal"); - - b.Property("SourceId") - .HasColumnType("uuid") - .HasColumnName("source_id"); + .HasColumnName("workflow"); b.HasKey("Id") - .HasName("pk_secrets"); + .HasName("pk_workflow_snapshots"); - b.HasIndex("SourceId") - .HasDatabaseName("ix_secrets_source_id"); + b.HasIndex("CommitId") + .IsUnique() + .HasDatabaseName("ix_workflow_snapshots_commit_id"); - b.ToTable("secrets", (string)null); + b.ToTable("workflow_snapshots", (string)null); }); - modelBuilder.Entity("marlin.core.models.Job", b => + modelBuilder.Entity("marlin.web.Models.Job", b => { - b.HasOne("marlin.core.models.Agent", "Agent") - .WithMany() + b.HasOne("marlin.web.Models.Agent", "Agent") + .WithMany("Jobs") .HasForeignKey("AgentId") .HasConstraintName("fk_jobs_agents_agent_id"); - b.HasOne("marlin.core.models.VersionControlSource", "Source") + b.HasOne("marlin.web.Models.VersionControlledBranch", "Branch") + .WithMany() + .HasForeignKey("BranchId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_branches_branch_id"); + + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithMany() + .HasForeignKey("CommitId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_jobs_source_commits_commit_id"); + + b.HasOne("marlin.web.Models.VersionControlSource", "Source") .WithMany() .HasForeignKey("SourceId") .OnDelete(DeleteBehavior.Cascade) @@ -324,12 +385,16 @@ namespace marlin.web.Migrations b.Navigation("Agent"); + b.Navigation("Branch"); + + b.Navigation("Commit"); + b.Navigation("Source"); }); - modelBuilder.Entity("marlin.core.models.JobStatusUpdate", b => + modelBuilder.Entity("marlin.web.Models.JobStatusUpdate", b => { - b.HasOne("marlin.core.models.Job", "Job") + b.HasOne("marlin.web.Models.Job", "Job") .WithMany("StatusUpdates") .HasForeignKey("JobId") .OnDelete(DeleteBehavior.Cascade) @@ -339,16 +404,28 @@ namespace marlin.web.Migrations b.Navigation("Job"); }); - modelBuilder.Entity("marlin.core.models.VersionControlledBlob", b => + modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => { - b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + b.HasOne("marlin.web.Models.VersionControlSource", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_secrets_sources_source_id"); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("marlin.web.Models.VersionControlledBlob", b => + { + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") .WithMany("Blobs") .HasForeignKey("CommitId") .OnDelete(DeleteBehavior.Cascade) .IsRequired() .HasConstraintName("fk_source_blobs_source_commits_commit_id"); - b.HasOne("marlin.core.models.VersionControlSource", "Source") + b.HasOne("marlin.web.Models.VersionControlSource", "Source") .WithMany() .HasForeignKey("SourceId") .OnDelete(DeleteBehavior.Cascade) @@ -360,16 +437,16 @@ namespace marlin.web.Migrations b.Navigation("Source"); }); - modelBuilder.Entity("marlin.core.models.VersionControlledBranch", b => + modelBuilder.Entity("marlin.web.Models.VersionControlledBranch", b => { - b.HasOne("marlin.core.models.VersionControlledCommit", "Commit") + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") .WithMany("Branches") .HasForeignKey("CommitId") .OnDelete(DeleteBehavior.Cascade) .IsRequired() .HasConstraintName("fk_source_branches_source_commits_commit_id"); - b.HasOne("marlin.core.models.VersionControlSource", "Source") + b.HasOne("marlin.web.Models.VersionControlSource", "Source") .WithMany("Branches") .HasForeignKey("SourceId") .OnDelete(DeleteBehavior.Cascade) @@ -381,9 +458,9 @@ namespace marlin.web.Migrations b.Navigation("Source"); }); - modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => { - b.HasOne("marlin.core.models.VersionControlSource", "Source") + b.HasOne("marlin.web.Models.VersionControlSource", "Source") .WithMany() .HasForeignKey("SourceId") .OnDelete(DeleteBehavior.Cascade) @@ -393,33 +470,40 @@ namespace marlin.web.Migrations b.Navigation("Source"); }); - modelBuilder.Entity("marlin.web.Models.VersionControlSecret", b => + modelBuilder.Entity("marlin.web.Models.WorkflowSnapshot", b => { - b.HasOne("marlin.core.models.VersionControlSource", "Source") - .WithMany() - .HasForeignKey("SourceId") + b.HasOne("marlin.web.Models.VersionControlledCommit", "Commit") + .WithOne("WorkflowSnapshot") + .HasForeignKey("marlin.web.Models.WorkflowSnapshot", "CommitId") .OnDelete(DeleteBehavior.Cascade) .IsRequired() - .HasConstraintName("fk_secrets_sources_source_id"); + .HasConstraintName("fk_workflow_snapshots_source_commits_commit_id"); - b.Navigation("Source"); + b.Navigation("Commit"); }); - modelBuilder.Entity("marlin.core.models.Job", b => + modelBuilder.Entity("marlin.web.Models.Agent", b => + { + b.Navigation("Jobs"); + }); + + modelBuilder.Entity("marlin.web.Models.Job", b => { b.Navigation("StatusUpdates"); }); - modelBuilder.Entity("marlin.core.models.VersionControlSource", b => + modelBuilder.Entity("marlin.web.Models.VersionControlSource", b => { b.Navigation("Branches"); }); - modelBuilder.Entity("marlin.core.models.VersionControlledCommit", b => + modelBuilder.Entity("marlin.web.Models.VersionControlledCommit", b => { b.Navigation("Blobs"); b.Navigation("Branches"); + + b.Navigation("WorkflowSnapshot"); }); #pragma warning restore 612, 618 } diff --git a/src/marlin.web/Models/Agent.cs b/src/marlin.web/Models/Agent.cs new file mode 100644 index 0000000..1faea9d --- /dev/null +++ b/src/marlin.web/Models/Agent.cs @@ -0,0 +1,17 @@ +namespace marlin.web.Models; + +using core.models; + +public class Agent +{ + public Guid Id { get; set; } + public string AuthToken { get; set; } + + public string Name { get; set; } + public int MaxConcurrentJobs { get; set; } + public DateTime LastSeen { get; set; } + + public CapabilitySet Capabilities { get; set; } + + public ICollection Jobs { get; set; } +} \ No newline at end of file diff --git a/src/marlin.web/Models/Job.cs b/src/marlin.web/Models/Job.cs new file mode 100644 index 0000000..bd60773 --- /dev/null +++ b/src/marlin.web/Models/Job.cs @@ -0,0 +1,48 @@ +namespace marlin.web.Models; + +public class Job +{ + public Guid Id { get; set; } + + public Guid SourceId { get; set; } + public VersionControlSource Source { get; set; } + + public Guid BranchId { get; set; } + public VersionControlledBranch Branch { get; set; } + + public Guid CommitId { get; set; } + public VersionControlledCommit Commit { get; set; } + + public string CreatedReason { get; set; } + public DateTime CreatedAt { get; set; } + public JobStatus Status { get; set; } + public ICollection StatusUpdates { get; set; } + + public Guid? AgentId { get; set; } + public Agent? Agent { get; set; } + + public string? ClaimToken { get; set; } + + public string Recipe { get; set; } + // plan +} + +public class JobStatusUpdate +{ + public Guid Id { get; set; } + + public Guid JobId { get; set; } + public Job Job { get; set; } + + public DateTime At { get; set; } + public JobStatus Status { get; set; } +} + +public enum JobStatus +{ + Pending, + Active, + Cancelled, + Success, + Failed +} \ No newline at end of file diff --git a/src/marlin.web/Models/SecretKind.cs b/src/marlin.web/Models/SecretKind.cs new file mode 100644 index 0000000..02652d5 --- /dev/null +++ b/src/marlin.web/Models/SecretKind.cs @@ -0,0 +1,7 @@ +namespace marlin.web.Models; + +public static class SecretKind +{ + public const string HTTPToken = "http-token"; + public const string SSHKey = "ssh-key"; +} \ No newline at end of file diff --git a/src/marlin.web/Models/VersionControlSource.cs b/src/marlin.web/Models/VersionControlSource.cs new file mode 100644 index 0000000..270df12 --- /dev/null +++ b/src/marlin.web/Models/VersionControlSource.cs @@ -0,0 +1,15 @@ +namespace marlin.web.Models; + +public class VersionControlSource +{ + public Guid Id { get; set; } + + public string UpstreamUrl { get; set; } + public string MainBranch { get; set; } + public string BranchFilter { get; set; } + public int MinPollingInterval { get; set; } + + public Guid? AccessSecretId { get; set; } + + public ICollection Branches { get; set; } +} \ No newline at end of file diff --git a/src/marlin.web/Models/VersionControlledBlob.cs b/src/marlin.web/Models/VersionControlledBlob.cs new file mode 100644 index 0000000..5b9017f --- /dev/null +++ b/src/marlin.web/Models/VersionControlledBlob.cs @@ -0,0 +1,15 @@ +namespace marlin.web.Models; + +public class VersionControlledBlob +{ + public Guid Id { get; set; } + + public Guid SourceId { get; set; } + public VersionControlSource Source { get; set; } + + public Guid CommitId { get; set; } + public VersionControlledCommit Commit { get; set; } + + public string Path { get; set; } + public byte[] Content { get; set; } +} \ No newline at end of file diff --git a/src/marlin.web/Models/VersionControlledBranch.cs b/src/marlin.web/Models/VersionControlledBranch.cs new file mode 100644 index 0000000..a2973f7 --- /dev/null +++ b/src/marlin.web/Models/VersionControlledBranch.cs @@ -0,0 +1,14 @@ +namespace marlin.web.Models; + +public class VersionControlledBranch +{ + public Guid Id { get; set; } + + public Guid SourceId { get; set; } + public VersionControlSource Source { get; set; } + + public Guid CommitId { get; set; } + public VersionControlledCommit Commit { get; set; } + + public string Name { get; set; } +} \ No newline at end of file diff --git a/src/marlin.web/Models/VersionControlledCommit.cs b/src/marlin.web/Models/VersionControlledCommit.cs new file mode 100644 index 0000000..9062156 --- /dev/null +++ b/src/marlin.web/Models/VersionControlledCommit.cs @@ -0,0 +1,17 @@ +namespace marlin.web.Models; + +public class VersionControlledCommit +{ + public Guid Id { get; set; } + + public Guid SourceId { get; set; } + public VersionControlSource Source { get; set; } + public bool BlobsFetched { get; set; } + public bool WorkflowScanned { get; set; } + + public string Hash { get; set; } + + public ICollection Branches { get; set; } + public ICollection Blobs { get; set; } + public WorkflowSnapshot? WorkflowSnapshot { get; set; } +} \ No newline at end of file diff --git a/src/marlin.web/Models/WorkflowSnapshot.cs b/src/marlin.web/Models/WorkflowSnapshot.cs new file mode 100644 index 0000000..2ea7bce --- /dev/null +++ b/src/marlin.web/Models/WorkflowSnapshot.cs @@ -0,0 +1,14 @@ +namespace marlin.web.Models; + +using marlin.core.models.meta; + +public class WorkflowSnapshot +{ + public Guid Id { get; set; } + + public Guid CommitId { get; set; } + public VersionControlledCommit Commit { get; set; } + + public DateTime ScannedAt { get; set; } + public Workflow Workflow { get; set; } +} diff --git a/src/marlin.web/Program.cs b/src/marlin.web/Program.cs index f079a3c..700e696 100644 --- a/src/marlin.web/Program.cs +++ b/src/marlin.web/Program.cs @@ -13,6 +13,16 @@ builder.Services.AddDbContext(options => options builder.Services.AddScoped(); +builder.Services + .AddAuthentication(AgentSchemeOptions.Name) + .AddScheme(AgentSchemeOptions.Name, null); + +builder.Services.AddAuthorization(options => +{ + options.AddPolicy("Agent", p => p.RequireClaim("agent_id")); + options.AddPolicy("JobScoped", p => p.RequireClaim("job_id")); +}); + builder.Services.AddHostedService(); builder.Services.AddSingleton(); @@ -20,7 +30,8 @@ builder.Services.AddHostedService(sp => sp .GetRequiredService()); // Add services to the container. -builder.Services.AddControllersWithViews(); +builder.Services.AddControllersWithViews(options => + options.OutputFormatters.RemoveType()); var app = builder.Build(); @@ -35,6 +46,7 @@ if (!app.Environment.IsDevelopment()) app.UseHttpsRedirection(); app.UseRouting(); +app.UseAuthentication(); app.UseAuthorization(); app.MapStaticAssets(); diff --git a/src/marlin.web/Properties/launchSettings.json b/src/marlin.web/Properties/launchSettings.json index b5224c4..e781de7 100644 --- a/src/marlin.web/Properties/launchSettings.json +++ b/src/marlin.web/Properties/launchSettings.json @@ -4,7 +4,7 @@ "http": { "commandName": "Project", "dotnetRunMessages": true, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "http://localhost:5090", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" @@ -13,7 +13,7 @@ "https": { "commandName": "Project", "dotnetRunMessages": true, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:7201;http://localhost:5090", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" diff --git a/src/marlin.web/Services/AgentAuthenticationHandler.cs b/src/marlin.web/Services/AgentAuthenticationHandler.cs new file mode 100644 index 0000000..519d109 --- /dev/null +++ b/src/marlin.web/Services/AgentAuthenticationHandler.cs @@ -0,0 +1,79 @@ +namespace marlin.web.Services; + +using System.Security.Claims; +using System.Text.Encodings.Web; +using Data; +using Microsoft.AspNetCore.Authentication; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Primitives; + +public class AgentSchemeOptions : AuthenticationSchemeOptions +{ + public const string Name = "Agent"; + public string AgentHeader { get; set; } = "Agent"; + public string JobHeader { get; set; } = "Job"; +} + +public class AgentAuthenticationHandler : AuthenticationHandler +{ + private readonly ApplicationDbContext _dbContext; + + public AgentAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder, + ApplicationDbContext dbContext) : base(options, logger, encoder) + { + this._dbContext = dbContext; + } + + private bool TryReadPair(string headerName, out Guid id, out string token) + { + id = Guid.Empty; + token = string.Empty; + + if (!Request.Headers.TryGetValue(headerName, out StringValues header)) + return false; + + var parts = header.ToString().Split(' ', 2); + if (parts.Length != 2 || parts[1].Length == 0) + return false; + + token = parts[1]; + return Guid.TryParse(parts[0], out id); + } + + protected override async Task HandleAuthenticateAsync() + { + var claims = new List(); + + if (!TryReadPair(Options.AgentHeader, out var agentId, out var agentToken)) + return AuthenticateResult.NoResult(); + + var agent = await this._dbContext.Agents + .FirstOrDefaultAsync(x => x.Id == agentId && x.AuthToken == agentToken); + + if (agent == null) + return AuthenticateResult.Fail("Invalid agent credentials."); + + claims.Add(new Claim("agent_id", agent.Id.ToString())); + + if (TryReadPair(Options.JobHeader, out var jobId, out var jobToken)) + { + var job = await this._dbContext.Jobs + .FirstOrDefaultAsync(x => x.Id == jobId && x.AgentId == agent.Id && x.ClaimToken == jobToken); + + if (job == null) + return AuthenticateResult.Fail("Invalid job claim token."); + + claims.Add(new Claim("job_id", job.Id.ToString())); + } + + var identity = new ClaimsIdentity(claims, this.Scheme.Name); + var principal = new ClaimsPrincipal(identity); + var ticket = new AuthenticationTicket(principal, this.Scheme.Name); + + return AuthenticateResult.Success(ticket); + } +} diff --git a/src/marlin.web/Services/VersionControlSyncService.cs b/src/marlin.web/Services/VersionControlSyncService.cs index bf189b0..cd75ed6 100644 --- a/src/marlin.web/Services/VersionControlSyncService.cs +++ b/src/marlin.web/Services/VersionControlSyncService.cs @@ -207,8 +207,9 @@ public class VersionControlSyncService : BackgroundService foreach (VersionControlledCommit commit in refs.Select(x => x.Commit)) { - var blobNames = await client.ListTree(localPath, commit.Hash, "."); - + var blobNames = await client.ListTree(localPath, commit.Hash, ".marlin"); + var workflowFiles = new List(); + foreach (string blobName in blobNames) { var content = await client.CatFile(localPath, commit.Hash, blobName); @@ -219,9 +220,40 @@ public class VersionControlSyncService : BackgroundService Path = blobName, Content = content }); + + if (blobName.EndsWith(".yml", StringComparison.OrdinalIgnoreCase) || + blobName.EndsWith(".yaml", StringComparison.OrdinalIgnoreCase)) + workflowFiles.Add(Encoding.UTF8.GetString(content)); } - + commit.BlobsFetched = true; + + this.ScanWorkflow(dbContext, commit, workflowFiles); + } + } + + private void ScanWorkflow(ApplicationDbContext dbContext, VersionControlledCommit commit, List workflowFiles) + { + try + { + var builder = new WorkflowBuilder(); + foreach (var file in workflowFiles) + builder.LoadFromString(file); + + dbContext.WorkflowSnapshots.Add(new WorkflowSnapshot + { + Commit = commit, + ScannedAt = DateTime.UtcNow, + Workflow = builder.Build() + }); + } + catch (Exception ex) + { + this._logger.LogWarning(ex, "Failed to parse workflow for commit '{hash}'", commit.Hash); + } + finally + { + commit.WorkflowScanned = true; } } } \ No newline at end of file diff --git a/src/marlin.web/Util/ClaimsPrincipalExtensions.cs b/src/marlin.web/Util/ClaimsPrincipalExtensions.cs new file mode 100644 index 0000000..3f4c307 --- /dev/null +++ b/src/marlin.web/Util/ClaimsPrincipalExtensions.cs @@ -0,0 +1,22 @@ +namespace marlin.web.Util; + +using System.Security.Claims; + +public static class ClaimsPrincipalExtensions +{ + public static Guid? GetAgentId(this ClaimsPrincipal user) => user.GetGuidClaim("agent_id"); + public static Guid? GetJobId(this ClaimsPrincipal user) => user.GetGuidClaim("job_id"); + public static Guid? GetUserId(this ClaimsPrincipal user) => user.GetGuidClaim("user_id"); + + public static Guid RequireAgentId(this ClaimsPrincipal user) => + user.GetAgentId() ?? throw new InvalidOperationException("Request has no agent_id claim."); + + public static Guid RequireJobId(this ClaimsPrincipal user) => + user.GetJobId() ?? throw new InvalidOperationException("Request has no job_id claim."); + + public static Guid RequireUserId(this ClaimsPrincipal user) => + user.GetUserId() ?? throw new InvalidOperationException("Request has no user_id claim."); + + private static Guid? GetGuidClaim(this ClaimsPrincipal user, string type) => + Guid.TryParse(user.FindFirstValue(type), out var value) ? value : null; +} diff --git a/src/marlin.web/Views/Agents/Create.cshtml b/src/marlin.web/Views/Agents/Create.cshtml new file mode 100644 index 0000000..4f0f31a --- /dev/null +++ b/src/marlin.web/Views/Agents/Create.cshtml @@ -0,0 +1,9 @@ +@using marlin.core.models + +
+ Name:
+ Max Concurrent Jobs:
+ +
+ +Back diff --git a/src/marlin.web/Views/Agents/Details.cshtml b/src/marlin.web/Views/Agents/Details.cshtml new file mode 100644 index 0000000..3512a98 --- /dev/null +++ b/src/marlin.web/Views/Agents/Details.cshtml @@ -0,0 +1,67 @@ +@using marlin.core.models +@model Agent + +
+
+

Details

+
+
+
+
Name
+
+ +
+ +
Max Concurrent Jobs
+
+ +
+ +
Last Seen
+
+ @Model.LastSeen.ToString("G") +
+ + +
+
+
+ +
+

Setup

+
+
+
+
Agent ID
+
+ @Model.Id +
+ +
Auth Token
+
+ @Model.AuthToken +
+ + +
+
+
+
+ +

Capabilities

+
+ + + @foreach (var capability in Model.Capabilities.Items) + { + + + + + } + +
@capability.Key@capability.Value
+ +Active + jobs +Back diff --git a/src/marlin.web/Views/Agents/List.cshtml b/src/marlin.web/Views/Agents/List.cshtml new file mode 100644 index 0000000..cb0e0df --- /dev/null +++ b/src/marlin.web/Views/Agents/List.cshtml @@ -0,0 +1,32 @@ +@model ICollection + +New + + + + + + + + + + + @foreach (var agent in Model) + { + + + + + + } + +
nameseenstatus
@agent.Name@agent.LastSeen.ToString("G") + @if (agent.Jobs.Any()) + { + processing @agent.Jobs.Count jobs + } + else + { + idling + } +
\ No newline at end of file diff --git a/src/marlin.web/Views/Jobs/Details.cshtml b/src/marlin.web/Views/Jobs/Details.cshtml index e1fd184..ebba94d 100644 --- a/src/marlin.web/Views/Jobs/Details.cshtml +++ b/src/marlin.web/Views/Jobs/Details.cshtml @@ -1,4 +1,4 @@ -@model marlin.core.models.Job +@model Job @Model.Id diff --git a/src/marlin.web/Views/Jobs/List.cshtml b/src/marlin.web/Views/Jobs/List.cshtml index 6ae1d96..968a94f 100644 --- a/src/marlin.web/Views/Jobs/List.cshtml +++ b/src/marlin.web/Views/Jobs/List.cshtml @@ -1,11 +1,10 @@ -@model ICollection +@model ICollection Spawn new... - @@ -19,12 +18,11 @@ @foreach (var source in Model) { - - + - + diff --git a/src/marlin.web/Views/Jobs/Spawn.cshtml b/src/marlin.web/Views/Jobs/Spawn.cshtml index 8f7ee43..848cae4 100644 --- a/src/marlin.web/Views/Jobs/Spawn.cshtml +++ b/src/marlin.web/Views/Jobs/Spawn.cshtml @@ -9,15 +9,21 @@ id = s.Id, label = s.UpstreamUrl, mainBranch = s.MainBranch, - branches = s.Branches.Select(b => new { name = b.Name, commit = b.Commit.Hash }) + branches = s.Branches.Select(b => new + { + id = b.Id, + name = b.Name, + commit = b.Commit.Hash, + recipes = b.Commit.WorkflowSnapshot?.Workflow.Recipes.Values.Select(r => new { id = r.Id, name = r.Name }) + ?? Enumerable.Empty() + }) })); }
VCS:
- Ref:
- Commit:
- Recipe:
+ Ref:
+ Recipe:
@@ -25,7 +31,7 @@ const sources = @Html.Raw(sourcesJson); const vcsSelect = document.getElementById('vcs'); const refSelect = document.getElementById('ref'); - const commitInput = document.getElementById('commit'); + const recipeSelect = document.getElementById('recipe'); for (const s of sources) { vcsSelect.add(new Option(s.label, s.id)); @@ -35,22 +41,29 @@ return sources.find(s => s.id === vcsSelect.value); } + function currentBranch() { + return currentSource()?.branches.find(b => b.id === refSelect.value); + } + function populateRefs() { refSelect.innerHTML = ''; for (const b of currentSource()?.branches ?? []) { - refSelect.add(new Option(b.name, b.name)); + refSelect.add(new Option(b.name, b.id)); } - refSelect.value = currentSource()?.mainBranch ?? ''; - populateCommit(); + const mainBranch = currentSource()?.branches.find(b => b.name === currentSource()?.mainBranch); + refSelect.value = mainBranch?.id ?? ''; + populateRecipes(); } - function populateCommit() { - const branch = currentSource()?.branches.find(b => b.name === refSelect.value); - commitInput.value = branch?.commit ?? ''; + function populateRecipes() { + recipeSelect.innerHTML = ''; + for (const r of currentBranch()?.recipes ?? []) { + recipeSelect.add(new Option(r.name, r.id)); + } } vcsSelect.addEventListener('change', populateRefs); - refSelect.addEventListener('change', populateCommit); + refSelect.addEventListener('change', populateRecipes); populateRefs(); diff --git a/src/marlin.web/Views/Shared/_Layout.cshtml b/src/marlin.web/Views/Shared/_Layout.cshtml index e8be2af..8ece676 100644 --- a/src/marlin.web/Views/Shared/_Layout.cshtml +++ b/src/marlin.web/Views/Shared/_Layout.cshtml @@ -29,6 +29,9 @@
  • Jobs
    • +
  • + Agents +
    • diff --git a/src/marlin.web/Views/VCS/Blobs.cshtml b/src/marlin.web/Views/VCS/Blobs.cshtml new file mode 100644 index 0000000..ab35260 --- /dev/null +++ b/src/marlin.web/Views/VCS/Blobs.cshtml @@ -0,0 +1,22 @@ +@using System.Text +@model ICollection + +
    link id status recipe
    open@source.Id@source.Id @source.Status @source.Recipe @source.Source.UpstreamUrl@source.Ref@source.Branch.Name @(source.Agent?.Name ?? "no agent assigned") @(source.CreatedReason)
    + + + + + + + + @foreach (var source in Model) + { + + + + + } + +
    pathsize
    @source.Path@source.Content.Length bytes
    + +back diff --git a/src/marlin.web/Views/VCS/Details.cshtml b/src/marlin.web/Views/VCS/Details.cshtml index 45c1a8f..a9ef0ea 100644 --- a/src/marlin.web/Views/VCS/Details.cshtml +++ b/src/marlin.web/Views/VCS/Details.cshtml @@ -25,6 +25,7 @@ branch commit + status @@ -34,6 +35,17 @@ @(branch.Name == Model.Source.MainBranch ? "@" : "") @branch.Name @branch.Commit.Hash + + @if (branch.Commit.BlobsFetched) + { + blobs + } + + @if (branch.Commit.WorkflowScanned) + { + workflow + } + } diff --git a/src/marlin.web/Views/VCS/List.cshtml b/src/marlin.web/Views/VCS/List.cshtml index 4c248e4..f109c92 100644 --- a/src/marlin.web/Views/VCS/List.cshtml +++ b/src/marlin.web/Views/VCS/List.cshtml @@ -1,4 +1,4 @@ -@model ICollection +@model ICollection New diff --git a/src/marlin.web/Views/VCS/ListSecret.cshtml b/src/marlin.web/Views/VCS/ListSecret.cshtml index aa5f15c..25db7fe 100644 --- a/src/marlin.web/Views/VCS/ListSecret.cshtml +++ b/src/marlin.web/Views/VCS/ListSecret.cshtml @@ -1,4 +1,4 @@ -@model (marlin.core.models.VersionControlSource Source, List Secrets) +@model (VersionControlSource Source, List Secrets) New diff --git a/src/marlin.web/Views/VCS/Workflow.cshtml b/src/marlin.web/Views/VCS/Workflow.cshtml new file mode 100644 index 0000000..da32ab1 --- /dev/null +++ b/src/marlin.web/Views/VCS/Workflow.cshtml @@ -0,0 +1,18 @@ +@model WorkflowSnapshot + +

    Workflow snapshot for @Model.Commit.Hash

    + +@foreach (var recipe in Model.Workflow.Recipes) +{ +
    +

    @recipe.Value.Name

    +
      + @foreach (var step in recipe.Value.Steps) + { +
    1. @step.Value.Name
      2. + } +
    +
    +} + +back diff --git a/src/marlin.web/wwwroot/css/site.css b/src/marlin.web/wwwroot/css/site.css index c6ecfb7..fa66346 100644 --- a/src/marlin.web/wwwroot/css/site.css +++ b/src/marlin.web/wwwroot/css/site.css @@ -8,6 +8,28 @@ html { } } +input[readonly] { + background-color: lightgray; +} + +dl { + display: grid; + grid-template-columns: max-content auto; + grid-column-gap: 10px; +} + +dl > form { + display: contents; +} + +dt { + grid-column-start: 1; +} + +dd { + grid-column-start: 2; +} + .btn:focus, .btn:active:focus, .btn-link.nav-link:focus, .form-control:focus, .form-check-input:focus { box-shadow: 0 0 0 0.1rem white, 0 0 0 0.25rem #258cfb; } @@ -35,6 +57,15 @@ table { width: 100%; } +table.vertable td:first-child { + background-color: #f0f0f0; + font-weight: bold; +} + +table tr:hover { + background-color: #fafafa +} + th, td { border: 1px solid #ccc; text-align: left; -- cgit v1.3.1