refactor: cleanup; feat: connection to agent

19 files changed, 248 insertions, 0 deletions

diff --git a/src/marlin.core/models/meta/AbstractStep.cs b/src/marlin.core/models/meta/AbstractStep.cs
new file mode 100644
index 0000000..7d10643
--- /dev/null
+++ b/src/marlin.core/models/meta/AbstractStep.cs
@@ -0,0 +1,16 @@
+namespace marlin.core.models.meta;
+
+public abstract class AbstractStep
+{
+    protected AbstractStep(string id)
+    {
+        this.Id = id;
+    }
+
+    public string Id { get; }
+    public string? Name { get; set; } = null;
+    public string? Run { get; set; } = null;
+    public Dictionary<string, string> With { get; set; } = new();
+    public FailBehavior? Fail { get; set; } = null;
+    public abstract IEnumerable<AbstractStep> GetAncestorsAndThis(Workflow workflow);
+}
diff --git a/src/marlin.core/models/meta/Artifact.cs b/src/marlin.core/models/meta/Artifact.cs
new file mode 100644
index 0000000..dce25a6
--- /dev/null
+++ b/src/marlin.core/models/meta/Artifact.cs
@@ -0,0 +1,26 @@
+namespace marlin.core.models.meta;
+
+public class Artifact
+{
+    public Artifact(string id)
+    {
+        this.Id = id;
+    }
+
+    public string Id { get; }
+    public string Path { get; set; }
+    public string Type { get; set; }
+    public ArtifactHandle Handle { get; set; } = ArtifactHandle.Defer;
+
+    public bool RequiresOrchestratorUpload()
+    {
+        return this.Type?.StartsWith("test/") == true || this.Type?.StartsWith("logs/") == true;
+    }
+}
+
+public enum ArtifactHandle
+{
+    Immediate,
+    Step,
+    Defer
+}
\ No newline at end of file
diff --git a/src/marlin.core/models/meta/ArtifactHandler.cs b/src/marlin.core/models/meta/ArtifactHandler.cs
new file mode 100644
index 0000000..51b6519
--- /dev/null
+++ b/src/marlin.core/models/meta/ArtifactHandler.cs
@@ -0,0 +1,12 @@
+namespace marlin.core.models.meta;
+
+public class ArtifactHandler
+{
+    public ArtifactHandler(string id)
+    {
+        this.Id = id;
+    }
+
+    public string Id { get; }
+    public ArtifactHandlerStep? Handle { get; set; }
+}
diff --git a/src/marlin.core/models/meta/ArtifactHandlerStep.cs b/src/marlin.core/models/meta/ArtifactHandlerStep.cs
new file mode 100644
index 0000000..bfcc9b9
--- /dev/null
+++ b/src/marlin.core/models/meta/ArtifactHandlerStep.cs
@@ -0,0 +1,14 @@
+namespace marlin.core.models.meta;
+
+public class ArtifactHandlerStep : AbstractStep
+{
+    public ArtifactHandlerStep(string id) : base(id)
+    {
+    }
+
+    public CleanupStep? Cleanup { get; set; } = null;
+    public override IEnumerable<AbstractStep> GetAncestorsAndThis(Workflow workflow)
+    {
+        yield return this;
+    }
+}
diff --git a/src/marlin.core/models/meta/CleanupAfter.cs b/src/marlin.core/models/meta/CleanupAfter.cs
new file mode 100644
index 0000000..0ca8f5a
--- /dev/null
+++ b/src/marlin.core/models/meta/CleanupAfter.cs
@@ -0,0 +1,7 @@
+namespace marlin.core.models.meta;
+
+public enum CleanupAfter
+{
+    Step,
+    Recipe
+}
\ No newline at end of file
diff --git a/src/marlin.core/models/meta/CleanupStep.cs b/src/marlin.core/models/meta/CleanupStep.cs
new file mode 100644
index 0000000..b2d895b
--- /dev/null
+++ b/src/marlin.core/models/meta/CleanupStep.cs
@@ -0,0 +1,14 @@
+namespace marlin.core.models.meta;
+
+public class CleanupStep : AbstractStep
+{
+    public CleanupStep(string id) : base(id)
+    {
+    }
+
+    public CleanupAfter After { get; set; }
+    public override IEnumerable<AbstractStep> GetAncestorsAndThis(Workflow workflow)
+    {
+        yield return this;
+    }
+}
\ No newline at end of file
diff --git a/src/marlin.core/models/meta/FailBehavior.cs b/src/marlin.core/models/meta/FailBehavior.cs
new file mode 100644
index 0000000..6084f5d
--- /dev/null
+++ b/src/marlin.core/models/meta/FailBehavior.cs
@@ -0,0 +1,8 @@
+namespace marlin.core.models.meta;
+
+public enum FailBehavior
+{
+    Continue,
+    Stop,
+    StopAfter
+}
\ No newline at end of file
diff --git a/src/marlin.core/models/meta/Recipe.cs b/src/marlin.core/models/meta/Recipe.cs
new file mode 100644
index 0000000..33ed814
--- /dev/null
+++ b/src/marlin.core/models/meta/Recipe.cs
@@ -0,0 +1,36 @@
+namespace marlin.core.models.meta;
+
+public class Recipe
+{
+    public Recipe(string id)
+    {
+        this.Id = id;
+    }
+
+    public string Id { get; }
+    public string Name { get; set; }
+
+    public string? RunsOnId { get; set; }
+    public List<string> RequireIds { get; set; } = [];
+    public Dictionary<string, RecipeStep> Steps { get; set; } = new();
+    public Dictionary<string, string> Parameters { get; set; } = new();
+    public Dictionary<string, string> Environment { get; set; } = new();
+
+    public RequirementSet? GetRunsOn(Workflow workflow) =>
+        this.RunsOnId != null ? workflow.Requirements[this.RunsOnId] : null;
+
+    public IEnumerable<Recipe> GetRequires(Workflow workflow) =>
+        this.RequireIds.Select(id => workflow.Recipes[id]);
+
+    public Dictionary<string, string> CreateArguments()
+    {
+        Dictionary<string, string> args = new();
+        foreach (var parameter in this.Parameters)
+            args.Add(parameter.Key, parameter.Value);
+
+        foreach (var environment in this.Environment)
+            args.Add($"env.{environment.Key}", environment.Value);
+
+        return args;
+    }
+}
diff --git a/src/marlin.core/models/meta/RecipeStep.cs b/src/marlin.core/models/meta/RecipeStep.cs
new file mode 100644
index 0000000..afe77cf
--- /dev/null
+++ b/src/marlin.core/models/meta/RecipeStep.cs
@@ -0,0 +1,28 @@
+namespace marlin.core.models.meta;
+
+public class RecipeStep : AbstractStep
+{
+    public RecipeStep(string id) : base(id)
+    {
+    }
+
+    public string? BaseId { get; set; }
+    public CleanupStep? Cleanup { get; set; }
+    public Dictionary<string, Artifact> Artifacts { get; set; } = new();
+    public List<string> Ingest { get; set; } = new();
+
+    public RecipeStep? GetBase(Workflow workflow) =>
+        this.BaseId != null ? workflow.Steps[this.BaseId] : null;
+
+    public override IEnumerable<AbstractStep> GetAncestorsAndThis(Workflow workflow)
+    {
+        yield return this;
+
+        var step = this.GetBase(workflow);
+        while (step != null)
+        {
+            yield return step;
+            step = step.GetBase(workflow);
+        }
+    }
+}
diff --git a/src/marlin.core/models/meta/RequirementSet.cs b/src/marlin.core/models/meta/RequirementSet.cs
new file mode 100644
index 0000000..13a3daa
--- /dev/null
+++ b/src/marlin.core/models/meta/RequirementSet.cs
@@ -0,0 +1,13 @@
+namespace marlin.core.models.meta;
+
+public class RequirementSet
+{
+    public RequirementSet(string id)
+    {
+        this.Id = id;
+    }
+
+    public string Id { get; }
+    public string? Base { get; set; }
+    public Dictionary<string, string> Has { get; set; } = new();
+}
diff --git a/src/marlin.core/models/meta/Workflow.cs b/src/marlin.core/models/meta/Workflow.cs
new file mode 100644
index 0000000..d7a200a
--- /dev/null
+++ b/src/marlin.core/models/meta/Workflow.cs
@@ -0,0 +1,9 @@
+namespace marlin.core.models.meta;
+
+public class Workflow
+{
+    public Dictionary<string, Recipe> Recipes { get; set; } = new();
+    public Dictionary<string, RecipeStep> Steps { get; set; } = new();
+    public Dictionary<string, ArtifactHandler> ArtifactHandlers { get; set; } = new();
+    public Dictionary<string, RequirementSet> Requirements { get; set; } = new();
+}
diff --git a/src/marlin.core/models/meta/dto/ArtifactDto.cs b/src/marlin.core/models/meta/dto/ArtifactDto.cs
new file mode 100644
index 0000000..aa2db76
--- /dev/null
+++ b/src/marlin.core/models/meta/dto/ArtifactDto.cs
@@ -0,0 +1,8 @@
+namespace marlin.core.models.meta.dto;
+
+public sealed record ArtifactDto
+{
+    public string? Path { get; init; }
+    public string? As { get; init; }
+    public ArtifactHandle? Handle { get; init; }
+}
diff --git a/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs b/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs
new file mode 100644
index 0000000..0e51a52
--- /dev/null
+++ b/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs
@@ -0,0 +1,6 @@
+namespace marlin.core.models.meta.dto;
+
+public sealed record ArtifactHandlerDto
+{
+    public StepDtoBase? Handle { get; init; }
+}
diff --git a/src/marlin.core/models/meta/dto/CleanupStepDto.cs b/src/marlin.core/models/meta/dto/CleanupStepDto.cs
new file mode 100644
index 0000000..606619b
--- /dev/null
+++ b/src/marlin.core/models/meta/dto/CleanupStepDto.cs
@@ -0,0 +1,6 @@
+namespace marlin.core.models.meta.dto;
+
+public sealed record CleanupStepDto : StepDtoBase
+{
+    public CleanupAfter? After { get; init; }
+}
diff --git a/src/marlin.core/models/meta/dto/RecipeDto.cs b/src/marlin.core/models/meta/dto/RecipeDto.cs
new file mode 100644
index 0000000..5e311b9
--- /dev/null
+++ b/src/marlin.core/models/meta/dto/RecipeDto.cs
@@ -0,0 +1,11 @@
+namespace marlin.core.models.meta.dto;
+
+public sealed record RecipeDto
+{
+    public string? Name { get; init; }
+    public string? RunsOn { get; init; }
+    public List<string>? Requires { get; init; }
+    public Dictionary<string, StepDto>? Steps { get; init; }
+    public Dictionary<string, string>? Parameters { get; init; }
+    public Dictionary<string, string>? Environment { get; init; }
+}
diff --git a/src/marlin.core/models/meta/dto/RequirementSetDto.cs b/src/marlin.core/models/meta/dto/RequirementSetDto.cs
new file mode 100644
index 0000000..165ed66
--- /dev/null
+++ b/src/marlin.core/models/meta/dto/RequirementSetDto.cs
@@ -0,0 +1,7 @@
+namespace marlin.core.models.meta.dto;
+
+public sealed record RequirementSetDto
+{
+    public string? Base { get; init; }
+    public Dictionary<string, string>? Has { get; init; }
+}
diff --git a/src/marlin.core/models/meta/dto/StepDto.cs b/src/marlin.core/models/meta/dto/StepDto.cs
new file mode 100644
index 0000000..57e4092
--- /dev/null
+++ b/src/marlin.core/models/meta/dto/StepDto.cs
@@ -0,0 +1,9 @@
+namespace marlin.core.models.meta.dto;
+
+public sealed record StepDto : StepDtoBase
+{
+    public string? Base { get; init; }
+    public CleanupStepDto? Cleanup { get; init; }
+    public Dictionary<string, ArtifactDto>? Artifacts { get; init; }
+    public List<string>? Ingest { get; init; }
+}
diff --git a/src/marlin.core/models/meta/dto/StepDtoBase.cs b/src/marlin.core/models/meta/dto/StepDtoBase.cs
new file mode 100644
index 0000000..6f4e36e
--- /dev/null
+++ b/src/marlin.core/models/meta/dto/StepDtoBase.cs
@@ -0,0 +1,9 @@
+namespace marlin.core.models.meta.dto;
+
+public record StepDtoBase
+{
+    public string? Name { get; init; }
+    public string? Run { get; init; }
+    public Dictionary<string, string>? With { get; init; }
+    public FailBehavior? Fail { get; init; }
+}
diff --git a/src/marlin.core/models/meta/dto/WorkflowFileDto.cs b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs
new file mode 100644
index 0000000..10997d6
--- /dev/null
+++ b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs
@@ -0,0 +1,9 @@
+namespace marlin.core.models.meta.dto;
+
+public sealed record WorkflowFileDto
+{
+    public Dictionary<string, RecipeDto>? Recipes { get; init; }
+    public Dictionary<string, StepDto>? Steps { get; init; }
+    public Dictionary<string, RequirementSetDto>? Requirements { get; init; }
+    public Dictionary<string, ArtifactHandlerDto>? ArtifactHandlers { get; init; }
+}