From 1e892b09b46de9908ddbdc895031895101313a26 Mon Sep 17 00:00:00 2001 From: Melodi <62381002+Melodi17@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:40:13 +1000 Subject: refactor: initial artifact cleanup and implementation --- src/marlin.core/models/meta/Artifact.cs | 26 ---------------------- src/marlin.core/models/meta/ArtifactHandler.cs | 12 ---------- src/marlin.core/models/meta/ArtifactHandlerStep.cs | 14 ------------ src/marlin.core/models/meta/RecipeStep.cs | 3 +-- src/marlin.core/models/meta/dto/ArtifactDto.cs | 8 ------- .../models/meta/dto/ArtifactHandlerDto.cs | 6 ----- src/marlin.core/models/meta/dto/StepDto.cs | 2 +- src/marlin.core/models/meta/dto/WorkflowFileDto.cs | 1 - src/marlin.core/models/plan/CommandRole.cs | 3 +-- src/marlin.core/models/plan/ExecutionCommand.cs | 1 + src/marlin.core/models/plan/ExecutionPlan.cs | 15 ++++++------- 11 files changed, 11 insertions(+), 80 deletions(-) delete mode 100644 src/marlin.core/models/meta/Artifact.cs delete mode 100644 src/marlin.core/models/meta/ArtifactHandler.cs delete mode 100644 src/marlin.core/models/meta/ArtifactHandlerStep.cs delete mode 100644 src/marlin.core/models/meta/dto/ArtifactDto.cs delete mode 100644 src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs (limited to 'src/marlin.core/models') diff --git a/src/marlin.core/models/meta/Artifact.cs b/src/marlin.core/models/meta/Artifact.cs deleted file mode 100644 index dce25a6..0000000 --- a/src/marlin.core/models/meta/Artifact.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace marlin.core.models.meta; - -public class Artifact -{ - public Artifact(string id) - { - this.Id = id; - } - - public string Id { get; } - public string Path { get; set; } - public string Type { get; set; } - public ArtifactHandle Handle { get; set; } = ArtifactHandle.Defer; - - public bool RequiresOrchestratorUpload() - { - return this.Type?.StartsWith("test/") == true || this.Type?.StartsWith("logs/") == true; - } -} - -public enum ArtifactHandle -{ - Immediate, - Step, - Defer -} \ No newline at end of file diff --git a/src/marlin.core/models/meta/ArtifactHandler.cs b/src/marlin.core/models/meta/ArtifactHandler.cs deleted file mode 100644 index 51b6519..0000000 --- a/src/marlin.core/models/meta/ArtifactHandler.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace marlin.core.models.meta; - -public class ArtifactHandler -{ - public ArtifactHandler(string id) - { - this.Id = id; - } - - public string Id { get; } - public ArtifactHandlerStep? Handle { get; set; } -} diff --git a/src/marlin.core/models/meta/ArtifactHandlerStep.cs b/src/marlin.core/models/meta/ArtifactHandlerStep.cs deleted file mode 100644 index bfcc9b9..0000000 --- a/src/marlin.core/models/meta/ArtifactHandlerStep.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace marlin.core.models.meta; - -public class ArtifactHandlerStep : AbstractStep -{ - public ArtifactHandlerStep(string id) : base(id) - { - } - - public CleanupStep? Cleanup { get; set; } = null; - public override IEnumerable GetAncestorsAndThis(Workflow workflow) - { - yield return this; - } -} diff --git a/src/marlin.core/models/meta/RecipeStep.cs b/src/marlin.core/models/meta/RecipeStep.cs index afe77cf..5cb1f78 100644 --- a/src/marlin.core/models/meta/RecipeStep.cs +++ b/src/marlin.core/models/meta/RecipeStep.cs @@ -8,8 +8,7 @@ public class RecipeStep : AbstractStep public string? BaseId { get; set; } public CleanupStep? Cleanup { get; set; } - public Dictionary Artifacts { get; set; } = new(); - public List Ingest { get; set; } = new(); + public Dictionary Artifacts { get; set; } = new(); public RecipeStep? GetBase(Workflow workflow) => this.BaseId != null ? workflow.Steps[this.BaseId] : null; diff --git a/src/marlin.core/models/meta/dto/ArtifactDto.cs b/src/marlin.core/models/meta/dto/ArtifactDto.cs deleted file mode 100644 index aa2db76..0000000 --- a/src/marlin.core/models/meta/dto/ArtifactDto.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace marlin.core.models.meta.dto; - -public sealed record ArtifactDto -{ - public string? Path { get; init; } - public string? As { get; init; } - public ArtifactHandle? Handle { get; init; } -} diff --git a/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs b/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs deleted file mode 100644 index 0e51a52..0000000 --- a/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace marlin.core.models.meta.dto; - -public sealed record ArtifactHandlerDto -{ - public StepDtoBase? Handle { get; init; } -} diff --git a/src/marlin.core/models/meta/dto/StepDto.cs b/src/marlin.core/models/meta/dto/StepDto.cs index 57e4092..c9ffd82 100644 --- a/src/marlin.core/models/meta/dto/StepDto.cs +++ b/src/marlin.core/models/meta/dto/StepDto.cs @@ -4,6 +4,6 @@ public sealed record StepDto : StepDtoBase { public string? Base { get; init; } public CleanupStepDto? Cleanup { get; init; } - public Dictionary? Artifacts { get; init; } + public Dictionary? Artifacts { get; init; } public List? Ingest { get; init; } } diff --git a/src/marlin.core/models/meta/dto/WorkflowFileDto.cs b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs index 10997d6..2a8d626 100644 --- a/src/marlin.core/models/meta/dto/WorkflowFileDto.cs +++ b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs @@ -5,5 +5,4 @@ public sealed record WorkflowFileDto public Dictionary? Recipes { get; init; } public Dictionary? Steps { get; init; } public Dictionary? Requirements { get; init; } - public Dictionary? ArtifactHandlers { get; init; } } diff --git a/src/marlin.core/models/plan/CommandRole.cs b/src/marlin.core/models/plan/CommandRole.cs index 8005532..f47b51e 100644 --- a/src/marlin.core/models/plan/CommandRole.cs +++ b/src/marlin.core/models/plan/CommandRole.cs @@ -3,6 +3,5 @@ namespace marlin.core.models.plan; public enum CommandRole { Step = 0, - ArtifactHandler = 1, - Cleanup = 2 + Cleanup = 1 } \ No newline at end of file diff --git a/src/marlin.core/models/plan/ExecutionCommand.cs b/src/marlin.core/models/plan/ExecutionCommand.cs index 4bf19c3..9649eec 100644 --- a/src/marlin.core/models/plan/ExecutionCommand.cs +++ b/src/marlin.core/models/plan/ExecutionCommand.cs @@ -9,6 +9,7 @@ public class ExecutionCommand public required string Command; public required CommandRole Role; public required bool Deferred; + public required IReadOnlyDictionary Artifacts; public CommandResultSelector? ResultSelector; public required FailBehavior FailBehavior; diff --git a/src/marlin.core/models/plan/ExecutionPlan.cs b/src/marlin.core/models/plan/ExecutionPlan.cs index 24c40e4..fa84634 100644 --- a/src/marlin.core/models/plan/ExecutionPlan.cs +++ b/src/marlin.core/models/plan/ExecutionPlan.cs @@ -1,20 +1,19 @@ namespace marlin.core.models.plan; +using System.Collections.ObjectModel; using meta; using util; public class ExecutionPlan { - public IEnumerable Components => [..this._commands, ..this._deferredArtifactHandler, ..this._deferredCleanup]; + public IEnumerable Components => [..this._commands, ..this._deferredCleanup]; private readonly List _commands; - private readonly List _deferredArtifactHandler; private readonly List _deferredCleanup; public ExecutionPlan() { this._commands = new(); - this._deferredArtifactHandler = new(); this._deferredCleanup = new(); } @@ -31,10 +30,6 @@ public class ExecutionPlan this._deferredCleanup.Add(command); break; - case CommandRole.ArtifactHandler: - this._deferredArtifactHandler.Add(command); - break; - default: throw new InvalidOperationException($"Cannot defer {command.Role}"); } @@ -117,6 +112,10 @@ public class ExecutionPlan run = VariableExpander.Expand(run, args); + IReadOnlyDictionary artifacts = step is RecipeStep rs + ? rs.Artifacts + : ReadOnlyDictionary.Empty; + ExecutionCommand stepCommand = new ExecutionCommand { Id = step.Id, @@ -124,12 +123,12 @@ public class ExecutionPlan Command = run, Role = step switch { - ArtifactHandlerStep => CommandRole.ArtifactHandler, RecipeStep => CommandRole.Step, CleanupStep => CommandRole.Cleanup, _ => throw new ArgumentOutOfRangeException() }, FailBehavior = fail.Value, + Artifacts = artifacts, Deferred = defer }; -- cgit v1.3.1