From df0737e55afeb5c01eb39909605970d1d3a73ffc Mon Sep 17 00:00:00 2001 From: Melodi <62381002+Melodi17@users.noreply.github.com> Date: Sun, 13 Sep 2026 13:22:20 +1000 Subject: refactor: cleanup; feat: connection to agent --- src/marlin.core/functional/WorkflowBuilder.cs | 153 ++++++++++++++++++ src/marlin.core/functional/WorkflowJson.cs | 17 ++ src/marlin.core/marlin.core.csproj | 4 + src/marlin.core/models/Agent.cs | 12 -- src/marlin.core/models/AgentStatus.cs | 4 + src/marlin.core/models/CapabilitySet.cs | 7 + src/marlin.core/models/DispatchJob.cs | 9 ++ src/marlin.core/models/Job.cs | 42 ----- src/marlin.core/models/SecretKind.cs | 7 - src/marlin.core/models/VersionControlSource.cs | 15 -- src/marlin.core/models/VersionControlledBlob.cs | 15 -- src/marlin.core/models/VersionControlledBranch.cs | 14 -- src/marlin.core/models/VersionControlledCommit.cs | 15 -- src/marlin.core/models/meta/AbstractStep.cs | 16 ++ src/marlin.core/models/meta/Artifact.cs | 26 +++ src/marlin.core/models/meta/ArtifactHandler.cs | 12 ++ src/marlin.core/models/meta/ArtifactHandlerStep.cs | 14 ++ src/marlin.core/models/meta/CleanupAfter.cs | 7 + src/marlin.core/models/meta/CleanupStep.cs | 14 ++ src/marlin.core/models/meta/FailBehavior.cs | 8 + src/marlin.core/models/meta/Recipe.cs | 36 +++++ src/marlin.core/models/meta/RecipeStep.cs | 28 ++++ src/marlin.core/models/meta/RequirementSet.cs | 13 ++ src/marlin.core/models/meta/Workflow.cs | 9 ++ src/marlin.core/models/meta/dto/ArtifactDto.cs | 8 + .../models/meta/dto/ArtifactHandlerDto.cs | 6 + src/marlin.core/models/meta/dto/CleanupStepDto.cs | 6 + src/marlin.core/models/meta/dto/RecipeDto.cs | 11 ++ .../models/meta/dto/RequirementSetDto.cs | 7 + src/marlin.core/models/meta/dto/StepDto.cs | 9 ++ src/marlin.core/models/meta/dto/StepDtoBase.cs | 9 ++ src/marlin.core/models/meta/dto/WorkflowFileDto.cs | 9 ++ src/marlin.core/models/plan/CommandResult.cs | 17 ++ .../models/plan/CommandResultSelector.cs | 13 ++ src/marlin.core/models/plan/CommandRole.cs | 8 + src/marlin.core/models/plan/ExecutionCommand.cs | 15 ++ src/marlin.core/models/plan/ExecutionPlan.cs | 174 +++++++++++++++++++++ src/marlin.core/util/SecureTokenGenerator.cs | 16 ++ src/marlin.core/util/VariableExpander.cs | 140 +++++++++++++++++ src/marlin.core/util/VariableExpansionException.cs | 56 +++++++ 40 files changed, 881 insertions(+), 120 deletions(-) create mode 100644 src/marlin.core/functional/WorkflowBuilder.cs create mode 100644 src/marlin.core/functional/WorkflowJson.cs delete mode 100644 src/marlin.core/models/Agent.cs create mode 100644 src/marlin.core/models/AgentStatus.cs create mode 100644 src/marlin.core/models/DispatchJob.cs delete mode 100644 src/marlin.core/models/Job.cs delete mode 100644 src/marlin.core/models/SecretKind.cs delete mode 100644 src/marlin.core/models/VersionControlSource.cs delete mode 100644 src/marlin.core/models/VersionControlledBlob.cs delete mode 100644 src/marlin.core/models/VersionControlledBranch.cs delete mode 100644 src/marlin.core/models/VersionControlledCommit.cs create mode 100644 src/marlin.core/models/meta/AbstractStep.cs create mode 100644 src/marlin.core/models/meta/Artifact.cs create mode 100644 src/marlin.core/models/meta/ArtifactHandler.cs create mode 100644 src/marlin.core/models/meta/ArtifactHandlerStep.cs create mode 100644 src/marlin.core/models/meta/CleanupAfter.cs create mode 100644 src/marlin.core/models/meta/CleanupStep.cs create mode 100644 src/marlin.core/models/meta/FailBehavior.cs create mode 100644 src/marlin.core/models/meta/Recipe.cs create mode 100644 src/marlin.core/models/meta/RecipeStep.cs create mode 100644 src/marlin.core/models/meta/RequirementSet.cs create mode 100644 src/marlin.core/models/meta/Workflow.cs create mode 100644 src/marlin.core/models/meta/dto/ArtifactDto.cs create mode 100644 src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs create mode 100644 src/marlin.core/models/meta/dto/CleanupStepDto.cs create mode 100644 src/marlin.core/models/meta/dto/RecipeDto.cs create mode 100644 src/marlin.core/models/meta/dto/RequirementSetDto.cs create mode 100644 src/marlin.core/models/meta/dto/StepDto.cs create mode 100644 src/marlin.core/models/meta/dto/StepDtoBase.cs create mode 100644 src/marlin.core/models/meta/dto/WorkflowFileDto.cs create mode 100644 src/marlin.core/models/plan/CommandResult.cs create mode 100644 src/marlin.core/models/plan/CommandResultSelector.cs create mode 100644 src/marlin.core/models/plan/CommandRole.cs create mode 100644 src/marlin.core/models/plan/ExecutionCommand.cs create mode 100644 src/marlin.core/models/plan/ExecutionPlan.cs create mode 100644 src/marlin.core/util/SecureTokenGenerator.cs create mode 100644 src/marlin.core/util/VariableExpander.cs create mode 100644 src/marlin.core/util/VariableExpansionException.cs (limited to 'src/marlin.core') diff --git a/src/marlin.core/functional/WorkflowBuilder.cs b/src/marlin.core/functional/WorkflowBuilder.cs new file mode 100644 index 0000000..79da909 --- /dev/null +++ b/src/marlin.core/functional/WorkflowBuilder.cs @@ -0,0 +1,153 @@ +namespace marlin.core.functional; + +using models.meta; +using models.meta.dto; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +public class WorkflowBuilder +{ + private static readonly IDeserializer Deserializer = new DeserializerBuilder() + .WithNamingConvention(HyphenatedNamingConvention.Instance) + .WithEnumNamingConvention(HyphenatedNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + private readonly Workflow _workflow = new(); + + public Workflow Build() => this._workflow; + + public void LoadFromDirectory(string path) + { + var files = Directory.EnumerateFiles(path, "*.yml", SearchOption.AllDirectories); + + foreach (var file in files) + this.LoadFromString(File.ReadAllText(file)); + } + + public void LoadFromString(string content) + { + var file = Deserializer.Deserialize(content); + if (file == null) + return; + + this.ApplyRequirements(file.Requirements); + this.ApplyRecipes(file.Recipes); + this.ApplySteps(file.Steps); + this.ApplyArtifactHandlers(file.ArtifactHandlers); + } + + private void ApplyRequirements(Dictionary? requirements) + { + if (requirements == null) + return; + + foreach (var (id, dto) in requirements) + { + var requirementSet = new RequirementSet(id) + { + Base = dto.Base, + Has = dto.Has != null ? new Dictionary(dto.Has) : new() + }; + + this._workflow.Requirements[id] = requirementSet; + } + } + + private void ApplyRecipes(Dictionary? recipes) + { + if (recipes == null) + return; + + foreach (var (id, dto) in recipes) + { + var recipe = new Recipe(id) + { + Name = dto.Name ?? id, + RunsOnId = dto.RunsOn, + RequireIds = dto.Requires ?? [], + Parameters = dto.Parameters != null ? new Dictionary(dto.Parameters) : new(), + Environment = dto.Environment != null ? new Dictionary(dto.Environment) : new() + }; + + if (dto.Steps != null) + { + foreach (var (stepId, stepDto) in dto.Steps) + recipe.Steps[stepId] = this.BuildRecipeStep(stepId, stepDto); + } + + this._workflow.Recipes[id] = recipe; + } + } + + private void ApplySteps(Dictionary? steps) + { + if (steps == null) + return; + + foreach (var (id, dto) in steps) + this._workflow.Steps[id] = this.BuildRecipeStep(id, dto); + } + + private RecipeStep BuildRecipeStep(string id, StepDto dto) + { + var step = new RecipeStep(id); + ApplyAbstractStep(step, dto); + + step.BaseId = dto.Base; + step.Ingest = dto.Ingest ?? []; + + if (dto.Artifacts != null) + { + foreach (var (artifactId, artifactDto) in dto.Artifacts) + { + step.Artifacts[artifactId] = new Artifact(artifactId) + { + Path = artifactDto.Path!, + Type = artifactDto.As!, + Handle = artifactDto.Handle ?? ArtifactHandle.Defer + }; + } + } + + if (dto.Cleanup != null) + step.Cleanup = BuildCleanupStep(id + "/cleanup", dto.Cleanup); + + return step; + } + + private static CleanupStep BuildCleanupStep(string id, CleanupStepDto dto) + { + var step = new CleanupStep(id) { After = dto.After ?? CleanupAfter.Step }; + ApplyAbstractStep(step, dto); + return step; + } + + private void ApplyArtifactHandlers(Dictionary? handlers) + { + if (handlers == null) + return; + + foreach (var (id, dto) in handlers) + { + var handler = new ArtifactHandler(id); + + if (dto.Handle != null) + { + var handleStep = new ArtifactHandlerStep(id + "/handle"); + ApplyAbstractStep(handleStep, dto.Handle); + handler.Handle = handleStep; + } + + this._workflow.ArtifactHandlers[id] = handler; + } + } + + private static void ApplyAbstractStep(AbstractStep step, StepDtoBase dto) + { + step.Name = dto.Name; + step.Run = dto.Run; + step.With = dto.With != null ? new Dictionary(dto.With) : new(); + step.Fail = dto.Fail; + } +} diff --git a/src/marlin.core/functional/WorkflowJson.cs b/src/marlin.core/functional/WorkflowJson.cs new file mode 100644 index 0000000..011edae --- /dev/null +++ b/src/marlin.core/functional/WorkflowJson.cs @@ -0,0 +1,17 @@ +namespace marlin.core.functional; + +using System.Text.Json; +using System.Text.Json.Serialization; +using models.meta; + +public static class WorkflowJson +{ + private static readonly JsonSerializerOptions Options = new() + { + Converters = { new JsonStringEnumConverter() } + }; + + public static string Serialize(Workflow workflow) => JsonSerializer.Serialize(workflow, Options); + + public static Workflow Deserialize(string json) => JsonSerializer.Deserialize(json, Options)!; +} diff --git a/src/marlin.core/marlin.core.csproj b/src/marlin.core/marlin.core.csproj index 237d661..5b65fba 100644 --- a/src/marlin.core/marlin.core.csproj +++ b/src/marlin.core/marlin.core.csproj @@ -6,4 +6,8 @@ enable + + + + diff --git a/src/marlin.core/models/Agent.cs b/src/marlin.core/models/Agent.cs deleted file mode 100644 index 2903131..0000000 --- a/src/marlin.core/models/Agent.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace marlin.core.models; - -public class Agent -{ - public Guid Id { get; set; } - - public string Name { get; set; } - public int MaxConcurrentJobs { get; set; } - public DateTime LastSeen { get; set; } - - public CapabilitySet Capabilities { get; set; } -} \ No newline at end of file diff --git a/src/marlin.core/models/AgentStatus.cs b/src/marlin.core/models/AgentStatus.cs new file mode 100644 index 0000000..393062f --- /dev/null +++ b/src/marlin.core/models/AgentStatus.cs @@ -0,0 +1,4 @@ +namespace marlin.core.models; + +public record AgentStatus(CapabilitySet CapabilitySet); +public record AgentConfiguration(string Name, int MaxConcurrentJobs); \ No newline at end of file diff --git a/src/marlin.core/models/CapabilitySet.cs b/src/marlin.core/models/CapabilitySet.cs index ac7da46..553d746 100644 --- a/src/marlin.core/models/CapabilitySet.cs +++ b/src/marlin.core/models/CapabilitySet.cs @@ -3,4 +3,11 @@ namespace marlin.core.models; public class CapabilitySet { public Dictionary Items { get; set; } = new(); +} + +public static class KnownCapabilities +{ + public const string CoreCount = "cores"; + public const string Memory = "memory"; + public const string Architecture = "architecture"; } \ No newline at end of file diff --git a/src/marlin.core/models/DispatchJob.cs b/src/marlin.core/models/DispatchJob.cs new file mode 100644 index 0000000..4070447 --- /dev/null +++ b/src/marlin.core/models/DispatchJob.cs @@ -0,0 +1,9 @@ +namespace marlin.core.models; + +public record DispatchJob( + Guid Id, + string Token, + string Vcs, + string Ref, + string Commit, + string Recipe); diff --git a/src/marlin.core/models/Job.cs b/src/marlin.core/models/Job.cs deleted file mode 100644 index a8c7c5d..0000000 --- a/src/marlin.core/models/Job.cs +++ /dev/null @@ -1,42 +0,0 @@ -namespace marlin.core.models; - -public class Job -{ - public Guid Id { get; set; } - - public Guid SourceId { get; set; } - public VersionControlSource Source { get; set; } - public string Ref { get; set; } - public string Commit { get; set; } - - public string CreatedReason { get; set; } - public DateTime CreatedAt { get; set; } - public JobStatus Status { get; set; } - public ICollection StatusUpdates { get; set; } - - public Guid? AgentId { get; set; } - public Agent? Agent { get; set; } - - public string Recipe { get; set; } - // plan -} - -public class JobStatusUpdate -{ - public Guid Id { get; set; } - - public Guid JobId { get; set; } - public Job Job { get; set; } - - public DateTime At { get; set; } - public JobStatus Status { get; set; } -} - -public enum JobStatus -{ - Pending, - Active, - Cancelled, - Success, - Failed -} \ No newline at end of file diff --git a/src/marlin.core/models/SecretKind.cs b/src/marlin.core/models/SecretKind.cs deleted file mode 100644 index 02652d5..0000000 --- a/src/marlin.core/models/SecretKind.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace marlin.web.Models; - -public static class SecretKind -{ - public const string HTTPToken = "http-token"; - public const string SSHKey = "ssh-key"; -} \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlSource.cs b/src/marlin.core/models/VersionControlSource.cs deleted file mode 100644 index daa8d51..0000000 --- a/src/marlin.core/models/VersionControlSource.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace marlin.core.models; - -public class VersionControlSource -{ - public Guid Id { get; set; } - - public string UpstreamUrl { get; set; } - public string MainBranch { get; set; } - public string BranchFilter { get; set; } - public int MinPollingInterval { get; set; } - - public Guid? AccessSecretId { get; set; } - - public ICollection Branches { get; set; } -} \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlledBlob.cs b/src/marlin.core/models/VersionControlledBlob.cs deleted file mode 100644 index 03915be..0000000 --- a/src/marlin.core/models/VersionControlledBlob.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace marlin.core.models; - -public class VersionControlledBlob -{ - public Guid Id { get; set; } - - public Guid SourceId { get; set; } - public VersionControlSource Source { get; set; } - - public Guid CommitId { get; set; } - public VersionControlledCommit Commit { get; set; } - - public string Path { get; set; } - public byte[] Content { get; set; } -} \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlledBranch.cs b/src/marlin.core/models/VersionControlledBranch.cs deleted file mode 100644 index 601f527..0000000 --- a/src/marlin.core/models/VersionControlledBranch.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace marlin.core.models; - -public class VersionControlledBranch -{ - public Guid Id { get; set; } - - public Guid SourceId { get; set; } - public VersionControlSource Source { get; set; } - - public Guid CommitId { get; set; } - public VersionControlledCommit Commit { get; set; } - - public string Name { get; set; } -} \ No newline at end of file diff --git a/src/marlin.core/models/VersionControlledCommit.cs b/src/marlin.core/models/VersionControlledCommit.cs deleted file mode 100644 index a7b5e9e..0000000 --- a/src/marlin.core/models/VersionControlledCommit.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace marlin.core.models; - -public class VersionControlledCommit -{ - public Guid Id { get; set; } - - public Guid SourceId { get; set; } - public VersionControlSource Source { get; set; } - public bool BlobsFetched { get; set; } - - public string Hash { get; set; } - - public ICollection Branches { get; set; } - public ICollection Blobs { get; set; } -} \ No newline at end of file diff --git a/src/marlin.core/models/meta/AbstractStep.cs b/src/marlin.core/models/meta/AbstractStep.cs new file mode 100644 index 0000000..7d10643 --- /dev/null +++ b/src/marlin.core/models/meta/AbstractStep.cs @@ -0,0 +1,16 @@ +namespace marlin.core.models.meta; + +public abstract class AbstractStep +{ + protected AbstractStep(string id) + { + this.Id = id; + } + + public string Id { get; } + public string? Name { get; set; } = null; + public string? Run { get; set; } = null; + public Dictionary With { get; set; } = new(); + public FailBehavior? Fail { get; set; } = null; + public abstract IEnumerable GetAncestorsAndThis(Workflow workflow); +} diff --git a/src/marlin.core/models/meta/Artifact.cs b/src/marlin.core/models/meta/Artifact.cs new file mode 100644 index 0000000..dce25a6 --- /dev/null +++ b/src/marlin.core/models/meta/Artifact.cs @@ -0,0 +1,26 @@ +namespace marlin.core.models.meta; + +public class Artifact +{ + public Artifact(string id) + { + this.Id = id; + } + + public string Id { get; } + public string Path { get; set; } + public string Type { get; set; } + public ArtifactHandle Handle { get; set; } = ArtifactHandle.Defer; + + public bool RequiresOrchestratorUpload() + { + return this.Type?.StartsWith("test/") == true || this.Type?.StartsWith("logs/") == true; + } +} + +public enum ArtifactHandle +{ + Immediate, + Step, + Defer +} \ No newline at end of file diff --git a/src/marlin.core/models/meta/ArtifactHandler.cs b/src/marlin.core/models/meta/ArtifactHandler.cs new file mode 100644 index 0000000..51b6519 --- /dev/null +++ b/src/marlin.core/models/meta/ArtifactHandler.cs @@ -0,0 +1,12 @@ +namespace marlin.core.models.meta; + +public class ArtifactHandler +{ + public ArtifactHandler(string id) + { + this.Id = id; + } + + public string Id { get; } + public ArtifactHandlerStep? Handle { get; set; } +} diff --git a/src/marlin.core/models/meta/ArtifactHandlerStep.cs b/src/marlin.core/models/meta/ArtifactHandlerStep.cs new file mode 100644 index 0000000..bfcc9b9 --- /dev/null +++ b/src/marlin.core/models/meta/ArtifactHandlerStep.cs @@ -0,0 +1,14 @@ +namespace marlin.core.models.meta; + +public class ArtifactHandlerStep : AbstractStep +{ + public ArtifactHandlerStep(string id) : base(id) + { + } + + public CleanupStep? Cleanup { get; set; } = null; + public override IEnumerable GetAncestorsAndThis(Workflow workflow) + { + yield return this; + } +} diff --git a/src/marlin.core/models/meta/CleanupAfter.cs b/src/marlin.core/models/meta/CleanupAfter.cs new file mode 100644 index 0000000..0ca8f5a --- /dev/null +++ b/src/marlin.core/models/meta/CleanupAfter.cs @@ -0,0 +1,7 @@ +namespace marlin.core.models.meta; + +public enum CleanupAfter +{ + Step, + Recipe +} \ No newline at end of file diff --git a/src/marlin.core/models/meta/CleanupStep.cs b/src/marlin.core/models/meta/CleanupStep.cs new file mode 100644 index 0000000..b2d895b --- /dev/null +++ b/src/marlin.core/models/meta/CleanupStep.cs @@ -0,0 +1,14 @@ +namespace marlin.core.models.meta; + +public class CleanupStep : AbstractStep +{ + public CleanupStep(string id) : base(id) + { + } + + public CleanupAfter After { get; set; } + public override IEnumerable GetAncestorsAndThis(Workflow workflow) + { + yield return this; + } +} \ No newline at end of file diff --git a/src/marlin.core/models/meta/FailBehavior.cs b/src/marlin.core/models/meta/FailBehavior.cs new file mode 100644 index 0000000..6084f5d --- /dev/null +++ b/src/marlin.core/models/meta/FailBehavior.cs @@ -0,0 +1,8 @@ +namespace marlin.core.models.meta; + +public enum FailBehavior +{ + Continue, + Stop, + StopAfter +} \ No newline at end of file diff --git a/src/marlin.core/models/meta/Recipe.cs b/src/marlin.core/models/meta/Recipe.cs new file mode 100644 index 0000000..33ed814 --- /dev/null +++ b/src/marlin.core/models/meta/Recipe.cs @@ -0,0 +1,36 @@ +namespace marlin.core.models.meta; + +public class Recipe +{ + public Recipe(string id) + { + this.Id = id; + } + + public string Id { get; } + public string Name { get; set; } + + public string? RunsOnId { get; set; } + public List RequireIds { get; set; } = []; + public Dictionary Steps { get; set; } = new(); + public Dictionary Parameters { get; set; } = new(); + public Dictionary Environment { get; set; } = new(); + + public RequirementSet? GetRunsOn(Workflow workflow) => + this.RunsOnId != null ? workflow.Requirements[this.RunsOnId] : null; + + public IEnumerable GetRequires(Workflow workflow) => + this.RequireIds.Select(id => workflow.Recipes[id]); + + public Dictionary CreateArguments() + { + Dictionary args = new(); + foreach (var parameter in this.Parameters) + args.Add(parameter.Key, parameter.Value); + + foreach (var environment in this.Environment) + args.Add($"env.{environment.Key}", environment.Value); + + return args; + } +} diff --git a/src/marlin.core/models/meta/RecipeStep.cs b/src/marlin.core/models/meta/RecipeStep.cs new file mode 100644 index 0000000..afe77cf --- /dev/null +++ b/src/marlin.core/models/meta/RecipeStep.cs @@ -0,0 +1,28 @@ +namespace marlin.core.models.meta; + +public class RecipeStep : AbstractStep +{ + public RecipeStep(string id) : base(id) + { + } + + public string? BaseId { get; set; } + public CleanupStep? Cleanup { get; set; } + public Dictionary Artifacts { get; set; } = new(); + public List Ingest { get; set; } = new(); + + public RecipeStep? GetBase(Workflow workflow) => + this.BaseId != null ? workflow.Steps[this.BaseId] : null; + + public override IEnumerable GetAncestorsAndThis(Workflow workflow) + { + yield return this; + + var step = this.GetBase(workflow); + while (step != null) + { + yield return step; + step = step.GetBase(workflow); + } + } +} diff --git a/src/marlin.core/models/meta/RequirementSet.cs b/src/marlin.core/models/meta/RequirementSet.cs new file mode 100644 index 0000000..13a3daa --- /dev/null +++ b/src/marlin.core/models/meta/RequirementSet.cs @@ -0,0 +1,13 @@ +namespace marlin.core.models.meta; + +public class RequirementSet +{ + public RequirementSet(string id) + { + this.Id = id; + } + + public string Id { get; } + public string? Base { get; set; } + public Dictionary Has { get; set; } = new(); +} diff --git a/src/marlin.core/models/meta/Workflow.cs b/src/marlin.core/models/meta/Workflow.cs new file mode 100644 index 0000000..d7a200a --- /dev/null +++ b/src/marlin.core/models/meta/Workflow.cs @@ -0,0 +1,9 @@ +namespace marlin.core.models.meta; + +public class Workflow +{ + public Dictionary Recipes { get; set; } = new(); + public Dictionary Steps { get; set; } = new(); + public Dictionary ArtifactHandlers { get; set; } = new(); + public Dictionary Requirements { get; set; } = new(); +} diff --git a/src/marlin.core/models/meta/dto/ArtifactDto.cs b/src/marlin.core/models/meta/dto/ArtifactDto.cs new file mode 100644 index 0000000..aa2db76 --- /dev/null +++ b/src/marlin.core/models/meta/dto/ArtifactDto.cs @@ -0,0 +1,8 @@ +namespace marlin.core.models.meta.dto; + +public sealed record ArtifactDto +{ + public string? Path { get; init; } + public string? As { get; init; } + public ArtifactHandle? Handle { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs b/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs new file mode 100644 index 0000000..0e51a52 --- /dev/null +++ b/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs @@ -0,0 +1,6 @@ +namespace marlin.core.models.meta.dto; + +public sealed record ArtifactHandlerDto +{ + public StepDtoBase? Handle { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/CleanupStepDto.cs b/src/marlin.core/models/meta/dto/CleanupStepDto.cs new file mode 100644 index 0000000..606619b --- /dev/null +++ b/src/marlin.core/models/meta/dto/CleanupStepDto.cs @@ -0,0 +1,6 @@ +namespace marlin.core.models.meta.dto; + +public sealed record CleanupStepDto : StepDtoBase +{ + public CleanupAfter? After { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/RecipeDto.cs b/src/marlin.core/models/meta/dto/RecipeDto.cs new file mode 100644 index 0000000..5e311b9 --- /dev/null +++ b/src/marlin.core/models/meta/dto/RecipeDto.cs @@ -0,0 +1,11 @@ +namespace marlin.core.models.meta.dto; + +public sealed record RecipeDto +{ + public string? Name { get; init; } + public string? RunsOn { get; init; } + public List? Requires { get; init; } + public Dictionary? Steps { get; init; } + public Dictionary? Parameters { get; init; } + public Dictionary? Environment { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/RequirementSetDto.cs b/src/marlin.core/models/meta/dto/RequirementSetDto.cs new file mode 100644 index 0000000..165ed66 --- /dev/null +++ b/src/marlin.core/models/meta/dto/RequirementSetDto.cs @@ -0,0 +1,7 @@ +namespace marlin.core.models.meta.dto; + +public sealed record RequirementSetDto +{ + public string? Base { get; init; } + public Dictionary? Has { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/StepDto.cs b/src/marlin.core/models/meta/dto/StepDto.cs new file mode 100644 index 0000000..57e4092 --- /dev/null +++ b/src/marlin.core/models/meta/dto/StepDto.cs @@ -0,0 +1,9 @@ +namespace marlin.core.models.meta.dto; + +public sealed record StepDto : StepDtoBase +{ + public string? Base { get; init; } + public CleanupStepDto? Cleanup { get; init; } + public Dictionary? Artifacts { get; init; } + public List? Ingest { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/StepDtoBase.cs b/src/marlin.core/models/meta/dto/StepDtoBase.cs new file mode 100644 index 0000000..6f4e36e --- /dev/null +++ b/src/marlin.core/models/meta/dto/StepDtoBase.cs @@ -0,0 +1,9 @@ +namespace marlin.core.models.meta.dto; + +public record StepDtoBase +{ + public string? Name { get; init; } + public string? Run { get; init; } + public Dictionary? With { get; init; } + public FailBehavior? Fail { get; init; } +} diff --git a/src/marlin.core/models/meta/dto/WorkflowFileDto.cs b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs new file mode 100644 index 0000000..10997d6 --- /dev/null +++ b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs @@ -0,0 +1,9 @@ +namespace marlin.core.models.meta.dto; + +public sealed record WorkflowFileDto +{ + public Dictionary? Recipes { get; init; } + public Dictionary? Steps { get; init; } + public Dictionary? Requirements { get; init; } + public Dictionary? ArtifactHandlers { get; init; } +} diff --git a/src/marlin.core/models/plan/CommandResult.cs b/src/marlin.core/models/plan/CommandResult.cs new file mode 100644 index 0000000..3385e84 --- /dev/null +++ b/src/marlin.core/models/plan/CommandResult.cs @@ -0,0 +1,17 @@ +namespace marlin.core.models.plan; + +[Flags] +public enum CommandResult : byte +{ + Success = 1, + Failure = 2, + Unconditional = Success | Failure, +} + +public static class CommandResultExtensions +{ + public static bool Matches(this CommandResult self, CommandResult contains) + { + return (self & contains) != 0; + } +} \ No newline at end of file diff --git a/src/marlin.core/models/plan/CommandResultSelector.cs b/src/marlin.core/models/plan/CommandResultSelector.cs new file mode 100644 index 0000000..92c41c7 --- /dev/null +++ b/src/marlin.core/models/plan/CommandResultSelector.cs @@ -0,0 +1,13 @@ +namespace marlin.core.models.plan; + +public class CommandResultSelector +{ + public ExecutionCommand Command { get; } + public CommandResult Result { get; } + + public CommandResultSelector(ExecutionCommand command, CommandResult result) + { + this.Command = command; + this.Result = result; + } +} \ No newline at end of file diff --git a/src/marlin.core/models/plan/CommandRole.cs b/src/marlin.core/models/plan/CommandRole.cs new file mode 100644 index 0000000..8005532 --- /dev/null +++ b/src/marlin.core/models/plan/CommandRole.cs @@ -0,0 +1,8 @@ +namespace marlin.core.models.plan; + +public enum CommandRole +{ + Step = 0, + ArtifactHandler = 1, + Cleanup = 2 +} \ No newline at end of file diff --git a/src/marlin.core/models/plan/ExecutionCommand.cs b/src/marlin.core/models/plan/ExecutionCommand.cs new file mode 100644 index 0000000..124da63 --- /dev/null +++ b/src/marlin.core/models/plan/ExecutionCommand.cs @@ -0,0 +1,15 @@ +namespace marlin.core.models.plan; + +using meta; + +public class ExecutionCommand +{ + public required string Id; + public required string Name; + public required string Command; + public required CommandRole Role; + public required bool Deferred; + public CommandResultSelector? ResultSelector; + + public required FailBehavior FailBehavior; +} \ No newline at end of file diff --git a/src/marlin.core/models/plan/ExecutionPlan.cs b/src/marlin.core/models/plan/ExecutionPlan.cs new file mode 100644 index 0000000..24c40e4 --- /dev/null +++ b/src/marlin.core/models/plan/ExecutionPlan.cs @@ -0,0 +1,174 @@ +namespace marlin.core.models.plan; + +using meta; +using util; + +public class ExecutionPlan +{ + public IEnumerable Components => [..this._commands, ..this._deferredArtifactHandler, ..this._deferredCleanup]; + + private readonly List _commands; + private readonly List _deferredArtifactHandler; + private readonly List _deferredCleanup; + + public ExecutionPlan() + { + this._commands = new(); + this._deferredArtifactHandler = new(); + this._deferredCleanup = new(); + } + + private void AddCommand(ExecutionCommand command) + { + if (!command.Deferred) + this._commands.Add(command); + + else + { + switch (command.Role) + { + case CommandRole.Cleanup: + this._deferredCleanup.Add(command); + break; + + case CommandRole.ArtifactHandler: + this._deferredArtifactHandler.Add(command); + break; + + default: + throw new InvalidOperationException($"Cannot defer {command.Role}"); + } + } + } + + public static ExecutionPlan Create(Workflow workflow, Recipe recipe) + { + ExecutionPlan plan = new(); + + AddToExecutionPlan(plan, workflow, recipe, null); + + return plan; + } + + private static ExecutionCommand? AddToExecutionPlan(ExecutionPlan plan, Workflow workflow, Recipe recipe, ExecutionCommand? previousCommand) + { + foreach (var dependentRecipe in recipe.GetRequires(workflow)) + previousCommand = AddToExecutionPlan(plan, workflow, dependentRecipe, previousCommand); + + Dictionary recipeArgs = recipe.CreateArguments(); + + foreach (var step in recipe.Steps.Values) + { + var command = CreateCommand(step, recipe, workflow, false, recipeArgs, out var commandArgs); + if (previousCommand != null) + { + command.ResultSelector = new CommandResultSelector(previousCommand, + previousCommand.FailBehavior switch + { + FailBehavior.Continue => CommandResult.Unconditional, + FailBehavior.Stop => CommandResult.Success, + FailBehavior.StopAfter => CommandResult.Success, + + _ => throw new ArgumentOutOfRangeException() + }); + } + + plan.AddCommand(command); + + var cleanupCommand = CreateCleanupCommand(step, command, recipe, workflow, commandArgs); + if (cleanupCommand != null) + plan.AddCommand(cleanupCommand); + + previousCommand = command; + } + + return previousCommand; + } + + private static ExecutionCommand CreateCommand( + AbstractStep step, + Recipe recipe, + Workflow workflow, + bool defer, + Dictionary baseArgs, + out Dictionary createdArgs) + { + string? name = null; + string? run = null; + FailBehavior? fail = null; + + Dictionary args = new(baseArgs); + + foreach (var stepHierarchy in step.GetAncestorsAndThis(workflow)) + { + name ??= stepHierarchy.Name; + run ??= stepHierarchy.Run; + fail ??= stepHierarchy.Fail; + + foreach (var kvp in stepHierarchy.With) + args.TryAdd(kvp.Key, kvp.Value); + } + + name ??= step.Id; + fail ??= FailBehavior.Stop; + + if (run == null) + throw new Exception($"No run found for {recipe.Name}.{name}"); + + run = VariableExpander.Expand(run, args); + + ExecutionCommand stepCommand = new ExecutionCommand + { + Id = step.Id, + Name = name, + Command = run, + Role = step switch + { + ArtifactHandlerStep => CommandRole.ArtifactHandler, + RecipeStep => CommandRole.Step, + CleanupStep => CommandRole.Cleanup, + _ => throw new ArgumentOutOfRangeException() + }, + FailBehavior = fail.Value, + Deferred = defer + }; + + createdArgs = args; + return stepCommand; + } + + private static ExecutionCommand? CreateCleanupCommand(RecipeStep parentStep, + ExecutionCommand parentCommand, + Recipe recipe, + Workflow workflow, + Dictionary baseArgs) + { + var cleanupStep = GetCleanup(parentStep, workflow); + if (cleanupStep == null) + return null; + + var cleanupCommand = CreateCommand(cleanupStep, recipe, workflow, + cleanupStep.After == CleanupAfter.Recipe, baseArgs, out _); + + cleanupCommand.ResultSelector = new CommandResultSelector(parentCommand, + parentCommand.FailBehavior switch + { + FailBehavior.Stop => CommandResult.Success, + FailBehavior.StopAfter => CommandResult.Unconditional, + FailBehavior.Continue => CommandResult.Unconditional, + + _ => throw new ArgumentOutOfRangeException() + }); + + return cleanupCommand; + } + + private static CleanupStep? GetCleanup(RecipeStep step, Workflow workflow) + { + return step + .GetAncestorsAndThis(workflow) + .OfType() + .Select(x => x.Cleanup) + .FirstOrDefault(x => x != null); + } +} diff --git a/src/marlin.core/util/SecureTokenGenerator.cs b/src/marlin.core/util/SecureTokenGenerator.cs new file mode 100644 index 0000000..f99918c --- /dev/null +++ b/src/marlin.core/util/SecureTokenGenerator.cs @@ -0,0 +1,16 @@ +namespace marlin.core.util; + +using System.Security.Cryptography; + +public class SecureTokenGenerator +{ + private const string ALLOWED_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + private static string Create(int length = 32) + { + return RandomNumberGenerator.GetString(ALLOWED_CHARS, length); + } + + public static string CreateAgentToken() => "ag-" + Create(); + public static string CreateClaimToken() => "cl-" + Create(); +} \ No newline at end of file diff --git a/src/marlin.core/util/VariableExpander.cs b/src/marlin.core/util/VariableExpander.cs new file mode 100644 index 0000000..3c3636e --- /dev/null +++ b/src/marlin.core/util/VariableExpander.cs @@ -0,0 +1,140 @@ +namespace marlin.core.util; + +using System.Text; +using System.Text.RegularExpressions; + +public class VariableExpander +{ + public static string Expand(string format, Dictionary variables) + { + StringBuilder output = new(); + + for (int i = 0; i < format.Length; i++) + { + char c = format[i]; + if (c == '{') + { + if (i + 1 >= format.Length) + throw new UnmatchedBraceException("Trailing expression '{'", i, format[i..]); + + if (format[i + 1] == '{') + { + output.Append('{'); + i++; + } + else + { + string expr = GetExpression(format, ref i); + string value = EvaluateExpression(expr, variables); + output.Append(value); + } + } + else if (c == '}') + { + if (i + 1 >= format.Length) + throw new UnmatchedBraceException("Uninitiated expression at '}'", i, format[i..]); + + if (format[i + 1] == '}') + { + output.Append('}'); + i++; + } + else + { + throw new UnmatchedBraceException("Uninitiated expression at '}'", i, format[i..]); + } + } + else + output.Append(c); + } + + string result = output.ToString(); + + // Clean up extra spaces: collapse multiple consecutive spaces (but preserve other whitespace) + result = Regex.Replace(result, @" {2,}", " "); + result = result.Trim(); + + return result; + } + + private static string GetExpression(string format, ref int pos) + { + StringBuilder part = new(); + + if (format[pos] != '{') + throw new InvalidExpressionException(format[pos..], "Expected '{' at start of expression"); + + pos++; + + for (; pos < format.Length; pos++) + { + if (format[pos] == '}') + break; + + part.Append(format[pos]); + } + + if (pos >= format.Length || format[pos] != '}') + throw new UnmatchedBraceException($"Trailing expression {format[pos..]}", pos, format[pos..]); + + return part.ToString(); + } + + private static string EvaluateExpression(string expr, Dictionary variables) + { + Match match; + + const string identifier = "[a-zA-Z0-9-_.]+"; + + // Variable value + match = Regex.Match(expr, $"^{identifier}$"); + if (match.Success) + return GetVariableOrThrow(expr, variables); + + // Conditional argument: only included if variable is set. + // --hello var + match = Regex.Match(expr, $"^(--?{identifier})\\s+({identifier})$"); + if (match.Success) + { + if (variables.TryGetValue(match.Groups[2].Value, out var value)) + return $"{match.Groups[1].Value} {value}"; + + else + return ""; + } + + // Default value, if value is unset, uses default + match = Regex.Match(expr, $@"^({identifier})\s*=\s*(.*)$"); + if (match.Success) + { + if (variables.TryGetValue(match.Groups[1].Value, out var value)) + return value; + else + return match.Groups[2].Value; + } + + // Use value if set + match = Regex.Match(expr, $@"^({identifier})\?\s*(.*)$"); + if (match.Success) + { + if (variables.TryGetValue(match.Groups[1].Value, out var value)) + { + if (string.IsNullOrEmpty(value) || value == "null" || value == "false") + return ""; + + return match.Groups[2].Value; + } + else + return ""; + } + + throw new InvalidExpressionException(expr); + } + + private static string GetVariableOrThrow(string variable, Dictionary variables) + { + return variables.TryGetValue(variable, out var value) + ? value + : throw new VariableNotFoundException(variable); + } +} \ No newline at end of file diff --git a/src/marlin.core/util/VariableExpansionException.cs b/src/marlin.core/util/VariableExpansionException.cs new file mode 100644 index 0000000..3053880 --- /dev/null +++ b/src/marlin.core/util/VariableExpansionException.cs @@ -0,0 +1,56 @@ +namespace marlin.core.util; + +/// +/// Base exception for all variable expansion errors +/// +public class VariableExpansionException(string message) : Exception(message); + +/// +/// Thrown when a variable is referenced but not found in the variables dictionary +/// +public class VariableNotFoundException : VariableExpansionException +{ + public string VariableName { get; } + + public VariableNotFoundException(string variableName) + : base($"Variable '{variableName}' not found") + { + this.VariableName = variableName; + } +} + +/// +/// Thrown when an expression has invalid syntax or cannot be parsed +/// +public class InvalidExpressionException : VariableExpansionException +{ + public string Expression { get; } + + public InvalidExpressionException(string expression) + : base($"Invalid expression '{expression}'") + { + this.Expression = expression; + } + + public InvalidExpressionException(string expression, string reason) + : base($"Invalid expression '{expression}': {reason}") + { + this.Expression = expression; + } +} + +/// +/// Thrown when braces are not properly matched or escaped +/// +public class UnmatchedBraceException : VariableExpansionException +{ + public int Position { get; } + public string Remainder { get; } + + public UnmatchedBraceException(string message, int position, string remainder) + : base(message) + { + this.Position = position; + this.Remainder = remainder; + } +} -- cgit v1.3.1