refactor: initial artifact cleanup and implementationmain

17 files changed, 144 insertions, 124 deletions

diff --git a/src/marlin.agents.shell/BlobArtifactStore.cs b/src/marlin.agents.shell/BlobArtifactStore.cs
new file mode 100644
index 0000000..e447c5a
--- /dev/null
+++ b/src/marlin.agents.shell/BlobArtifactStore.cs
@@ -0,0 +1,102 @@
+namespace marlin.agents.shell;
+
+using System.Security.Cryptography;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using core.util;
+using Microsoft.Extensions.FileSystemGlobbing;
+using Microsoft.Extensions.FileSystemGlobbing.Abstractions;
+
+public class BlobArtifactStore : IArtifactStore
+{
+    private static readonly string _storeDir = Path.Combine(CommonPaths.AgentRoot, "store");
+    private static readonly string _blobDir = Path.Combine(_storeDir, "blob");
+    private static readonly string _manifestFile = Path.Combine(_storeDir, "manifest.json");
+
+    private Dictionary<ArtifactManifestKey, ArtifactManifestEntry[]> _manifest
+    {
+        get => field != null ? field : throw new InvalidOperationException("Init not run, cannot access manifest");
+        set;
+    }
+
+    public void Init()
+    {
+        Directory.CreateDirectory(_storeDir);
+        Directory.CreateDirectory(_blobDir);
+
+        if (File.Exists(_manifestFile))
+        {
+            _manifest = JsonSerializer.Deserialize<Dictionary<ArtifactManifestKey, ArtifactManifestEntry[]>>(
+                File.ReadAllText(_manifestFile))
+                ?? throw new InvalidOperationException("Could not read manifest, possibly corrupted");
+        }
+        else
+            _manifest = new();
+    }
+    
+    public async Task Stow(Guid jobId, string stepId, string artifactId, string basePath, string glob)
+    {
+        var key = new ArtifactManifestKey { JobId = jobId, StepId = stepId, ArtifactId = artifactId };
+        var entries = new List<ArtifactManifestEntry>();
+
+        Matcher matcher = new();
+        matcher.AddInclude(glob);
+        var globMatches = matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo(basePath)));
+        
+        foreach (var globMatch in globMatches.Files)
+        {
+            var fullPath = Path.Join(basePath, globMatch.Path);
+            var entry = new ArtifactManifestEntry
+            {
+                RelativePath = globMatch.Path,
+                Hash = ComputeFileHash(fullPath)
+            };
+            
+            var targetPath = Path.Join(_blobDir, entry.Hash);
+            
+            if (!File.Exists(targetPath))
+                File.Move(fullPath, targetPath);
+            else
+                File.Delete(fullPath);
+            
+            entries.Add(entry);
+        }
+
+        _manifest[key] = entries.ToArray();
+        await File.WriteAllTextAsync(_manifestFile, JsonSerializer.Serialize(this._manifest));
+    }
+    
+    public async Task<IArtifactAccessHandle> Access(Guid jobId, string stepId, string artifactId)
+    {
+        throw new NotImplementedException();
+    }
+
+    private static string ComputeFileHash(string filePath)
+    {
+        using var stream = File.OpenRead(filePath);
+        byte[] hash = SHA256.HashData(stream);
+        return Convert.ToHexStringLower(hash);
+    }
+}
+
+public class BlobArtifactAccessHandle : IArtifactAccessHandle
+{
+    public string Path { get; }
+    
+    public void Dispose() => Directory.Delete(Path, true);
+}
+
+public class ArtifactManifestKey
+{
+    public required Guid JobId { get; init; }
+    public required string StepId { get; init; }
+    public required string ArtifactId { get; init; }
+
+    public override int GetHashCode() => HashCode.Combine(JobId.GetHashCode(), StepId.GetHashCode(), ArtifactId.GetHashCode());
+}
+
+public class ArtifactManifestEntry
+{
+    public required string RelativePath { get; init; }
+    public required string Hash { get; init; }
+}
\ No newline at end of file
diff --git a/src/marlin.agents.shell/IArtifactStore.cs b/src/marlin.agents.shell/IArtifactStore.cs
new file mode 100644
index 0000000..2d6338c
--- /dev/null
+++ b/src/marlin.agents.shell/IArtifactStore.cs
@@ -0,0 +1,15 @@
+namespace marlin.agents.shell;
+
+public interface IArtifactStore
+{
+    /// Extracts given artifacts into shared flat blob storage
+    Task Stow(Guid jobId, string stepId, string artifactId, string basePath, string glob);
+    
+    /// Constructs virtual directory structure to access a given artifact
+    Task<IArtifactAccessHandle> Access(Guid jobId, string stepId, string artifactId);
+}
+
+public interface IArtifactAccessHandle : IDisposable
+{
+    public string Path { get; }
+}
\ No newline at end of file
diff --git a/src/marlin.agents.shell/LocalInstanceService.cs b/src/marlin.agents.shell/LocalInstanceService.cs
new file mode 100644
index 0000000..d0d2521
--- /dev/null
+++ b/src/marlin.agents.shell/LocalInstanceService.cs
@@ -0,0 +1,13 @@
+using marlin.core.contracts;
+using ProtoBuf.Grpc;
+
+public class LocalInstanceService : IInstanceService
+{
+    public async Task SendLogEvent(LogEventRequest request, CallContext context = default)
+    {
+        TextWriter writer = request.Error ? Console.Error : Console.Out;
+        await writer.WriteLineAsync(request.Message);
+    }
+    
+    public Task SendStatusUpdate(StatusUpdateRequest request, CallContext context = default) => Task.CompletedTask;
+}
\ No newline at end of file
diff --git a/src/marlin.agents.shell/Program.cs b/src/marlin.agents.shell/Program.cs
index c895e8d..1a3e97f 100644
--- a/src/marlin.agents.shell/Program.cs
+++ b/src/marlin.agents.shell/Program.cs
@@ -2,13 +2,11 @@ using System.Runtime.InteropServices;
 using System.Threading.Channels;
 using Grpc.Net.Client;
 using marlin.agents.shell;
-using marlin.core.contracts;
 using marlin.core.functional;
 using marlin.core.models;
 using marlin.core.models.meta;
 using marlin.core.models.plan;
 using marlin.core.util;
-using ProtoBuf.Grpc;
 using ProtoBuf.Grpc.Client;
 
 DotNetEnv.Env.NoClobber().TraversePath().Load();
@@ -25,15 +23,4 @@ else
 
     JobUnitOfWork uow = new(new LocalInstanceService(), repoPath, recipe);
     await uow.Run();
-}
-
-public class LocalInstanceService : IInstanceService
-{
-    public async Task SendLogEvent(LogEventRequest request, CallContext context = default)
-    {
-        TextWriter writer = request.Error ? Console.Error : Console.Out;
-        await writer.WriteLineAsync(request.Message);
-    }
-    
-    public Task SendStatusUpdate(StatusUpdateRequest request, CallContext context = default) => Task.CompletedTask;
 }
\ No newline at end of file
diff --git a/src/marlin.core/functional/WorkflowBuilder.cs b/src/marlin.core/functional/WorkflowBuilder.cs
index 79da909..a912196 100644
--- a/src/marlin.core/functional/WorkflowBuilder.cs
+++ b/src/marlin.core/functional/WorkflowBuilder.cs
@@ -34,7 +34,6 @@ public class WorkflowBuilder
         this.ApplyRequirements(file.Requirements);
         this.ApplyRecipes(file.Recipes);
         this.ApplySteps(file.Steps);
-        this.ApplyArtifactHandlers(file.ArtifactHandlers);
     }
 
     private void ApplyRequirements(Dictionary<string, RequirementSetDto>? requirements)
@@ -95,19 +94,11 @@ public class WorkflowBuilder
         ApplyAbstractStep(step, dto);
 
         step.BaseId = dto.Base;
-        step.Ingest = dto.Ingest ?? [];
 
         if (dto.Artifacts != null)
         {
-            foreach (var (artifactId, artifactDto) in dto.Artifacts)
-            {
-                step.Artifacts[artifactId] = new Artifact(artifactId)
-                {
-                    Path = artifactDto.Path!,
-                    Type = artifactDto.As!,
-                    Handle = artifactDto.Handle ?? ArtifactHandle.Defer
-                };
-            }
+            foreach (var item in dto.Artifacts)
+                step.Artifacts[item.Key] = item.Value;
         }
 
         if (dto.Cleanup != null)
@@ -123,26 +114,6 @@ public class WorkflowBuilder
         return step;
     }
 
-    private void ApplyArtifactHandlers(Dictionary<string, ArtifactHandlerDto>? handlers)
-    {
-        if (handlers == null)
-            return;
-
-        foreach (var (id, dto) in handlers)
-        {
-            var handler = new ArtifactHandler(id);
-
-            if (dto.Handle != null)
-            {
-                var handleStep = new ArtifactHandlerStep(id + "/handle");
-                ApplyAbstractStep(handleStep, dto.Handle);
-                handler.Handle = handleStep;
-            }
-
-            this._workflow.ArtifactHandlers[id] = handler;
-        }
-    }
-
     private static void ApplyAbstractStep(AbstractStep step, StepDtoBase dto)
     {
         step.Name = dto.Name;
diff --git a/src/marlin.core/marlin.core.csproj b/src/marlin.core/marlin.core.csproj
index 044c698..7a1681c 100644
--- a/src/marlin.core/marlin.core.csproj
+++ b/src/marlin.core/marlin.core.csproj
@@ -7,6 +7,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="10.0.14" />
       <PackageReference Include="protobuf-net.Grpc" Version="1.3.14" />
       <PackageReference Include="YamlDotNet" Version="18.1.0" />
     </ItemGroup>
diff --git a/src/marlin.core/models/meta/Artifact.cs b/src/marlin.core/models/meta/Artifact.cs
deleted file mode 100644
index dce25a6..0000000
--- a/src/marlin.core/models/meta/Artifact.cs
+++ /dev/null
@@ -1,26 +0,0 @@
-namespace marlin.core.models.meta;
-
-public class Artifact
-{
-    public Artifact(string id)
-    {
-        this.Id = id;
-    }
-
-    public string Id { get; }
-    public string Path { get; set; }
-    public string Type { get; set; }
-    public ArtifactHandle Handle { get; set; } = ArtifactHandle.Defer;
-
-    public bool RequiresOrchestratorUpload()
-    {
-        return this.Type?.StartsWith("test/") == true || this.Type?.StartsWith("logs/") == true;
-    }
-}
-
-public enum ArtifactHandle
-{
-    Immediate,
-    Step,
-    Defer
-}
\ No newline at end of file
diff --git a/src/marlin.core/models/meta/ArtifactHandler.cs b/src/marlin.core/models/meta/ArtifactHandler.cs
deleted file mode 100644
index 51b6519..0000000
--- a/src/marlin.core/models/meta/ArtifactHandler.cs
+++ /dev/null
@@ -1,12 +0,0 @@
-namespace marlin.core.models.meta;
-
-public class ArtifactHandler
-{
-    public ArtifactHandler(string id)
-    {
-        this.Id = id;
-    }
-
-    public string Id { get; }
-    public ArtifactHandlerStep? Handle { get; set; }
-}
diff --git a/src/marlin.core/models/meta/ArtifactHandlerStep.cs b/src/marlin.core/models/meta/ArtifactHandlerStep.cs
deleted file mode 100644
index bfcc9b9..0000000
--- a/src/marlin.core/models/meta/ArtifactHandlerStep.cs
+++ /dev/null
@@ -1,14 +0,0 @@
-namespace marlin.core.models.meta;
-
-public class ArtifactHandlerStep : AbstractStep
-{
-    public ArtifactHandlerStep(string id) : base(id)
-    {
-    }
-
-    public CleanupStep? Cleanup { get; set; } = null;
-    public override IEnumerable<AbstractStep> GetAncestorsAndThis(Workflow workflow)
-    {
-        yield return this;
-    }
-}
diff --git a/src/marlin.core/models/meta/RecipeStep.cs b/src/marlin.core/models/meta/RecipeStep.cs
index afe77cf..5cb1f78 100644
--- a/src/marlin.core/models/meta/RecipeStep.cs
+++ b/src/marlin.core/models/meta/RecipeStep.cs
@@ -8,8 +8,7 @@ public class RecipeStep : AbstractStep
 
     public string? BaseId { get; set; }
     public CleanupStep? Cleanup { get; set; }
-    public Dictionary<string, Artifact> Artifacts { get; set; } = new();
-    public List<string> Ingest { get; set; } = new();
+    public Dictionary<string, string> Artifacts { get; set; } = new();
 
     public RecipeStep? GetBase(Workflow workflow) =>
         this.BaseId != null ? workflow.Steps[this.BaseId] : null;
diff --git a/src/marlin.core/models/meta/dto/ArtifactDto.cs b/src/marlin.core/models/meta/dto/ArtifactDto.cs
deleted file mode 100644
index aa2db76..0000000
--- a/src/marlin.core/models/meta/dto/ArtifactDto.cs
+++ /dev/null
@@ -1,8 +0,0 @@
-namespace marlin.core.models.meta.dto;
-
-public sealed record ArtifactDto
-{
-    public string? Path { get; init; }
-    public string? As { get; init; }
-    public ArtifactHandle? Handle { get; init; }
-}
diff --git a/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs b/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs
deleted file mode 100644
index 0e51a52..0000000
--- a/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs
+++ /dev/null
@@ -1,6 +0,0 @@
-namespace marlin.core.models.meta.dto;
-
-public sealed record ArtifactHandlerDto
-{
-    public StepDtoBase? Handle { get; init; }
-}
diff --git a/src/marlin.core/models/meta/dto/StepDto.cs b/src/marlin.core/models/meta/dto/StepDto.cs
index 57e4092..c9ffd82 100644
--- a/src/marlin.core/models/meta/dto/StepDto.cs
+++ b/src/marlin.core/models/meta/dto/StepDto.cs
@@ -4,6 +4,6 @@ public sealed record StepDto : StepDtoBase
 {
     public string? Base { get; init; }
     public CleanupStepDto? Cleanup { get; init; }
-    public Dictionary<string, ArtifactDto>? Artifacts { get; init; }
+    public Dictionary<string, string>? Artifacts { get; init; }
     public List<string>? Ingest { get; init; }
 }
diff --git a/src/marlin.core/models/meta/dto/WorkflowFileDto.cs b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs
index 10997d6..2a8d626 100644
--- a/src/marlin.core/models/meta/dto/WorkflowFileDto.cs
+++ b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs
@@ -5,5 +5,4 @@ public sealed record WorkflowFileDto
     public Dictionary<string, RecipeDto>? Recipes { get; init; }
     public Dictionary<string, StepDto>? Steps { get; init; }
     public Dictionary<string, RequirementSetDto>? Requirements { get; init; }
-    public Dictionary<string, ArtifactHandlerDto>? ArtifactHandlers { get; init; }
 }
diff --git a/src/marlin.core/models/plan/CommandRole.cs b/src/marlin.core/models/plan/CommandRole.cs
index 8005532..f47b51e 100644
--- a/src/marlin.core/models/plan/CommandRole.cs
+++ b/src/marlin.core/models/plan/CommandRole.cs
@@ -3,6 +3,5 @@ namespace marlin.core.models.plan;
 public enum CommandRole
 {
     Step = 0,
-    ArtifactHandler = 1,
-    Cleanup = 2
+    Cleanup = 1
 }
\ No newline at end of file
diff --git a/src/marlin.core/models/plan/ExecutionCommand.cs b/src/marlin.core/models/plan/ExecutionCommand.cs
index 4bf19c3..9649eec 100644
--- a/src/marlin.core/models/plan/ExecutionCommand.cs
+++ b/src/marlin.core/models/plan/ExecutionCommand.cs
@@ -9,6 +9,7 @@ public class ExecutionCommand
     public required string Command;
     public required CommandRole Role;
     public required bool Deferred;
+    public required IReadOnlyDictionary<string, string> Artifacts;
     public CommandResultSelector? ResultSelector;
 
     public required FailBehavior FailBehavior;
diff --git a/src/marlin.core/models/plan/ExecutionPlan.cs b/src/marlin.core/models/plan/ExecutionPlan.cs
index 24c40e4..fa84634 100644
--- a/src/marlin.core/models/plan/ExecutionPlan.cs
+++ b/src/marlin.core/models/plan/ExecutionPlan.cs
@@ -1,20 +1,19 @@
 namespace marlin.core.models.plan;
 
+using System.Collections.ObjectModel;
 using meta;
 using util;
 
 public class ExecutionPlan
 {
-    public IEnumerable<ExecutionCommand> Components => [..this._commands, ..this._deferredArtifactHandler, ..this._deferredCleanup];
+    public IEnumerable<ExecutionCommand> Components => [..this._commands, ..this._deferredCleanup];
 
     private readonly List<ExecutionCommand> _commands;
-    private readonly List<ExecutionCommand> _deferredArtifactHandler;
     private readonly List<ExecutionCommand> _deferredCleanup;
 
     public ExecutionPlan()
     {
         this._commands = new();
-        this._deferredArtifactHandler = new();
         this._deferredCleanup = new();
     }
 
@@ -31,10 +30,6 @@ public class ExecutionPlan
                     this._deferredCleanup.Add(command);
                     break;
 
-                case CommandRole.ArtifactHandler:
-                    this._deferredArtifactHandler.Add(command);
-                    break;
-
                 default:
                     throw new InvalidOperationException($"Cannot defer {command.Role}");
             }
@@ -117,6 +112,10 @@ public class ExecutionPlan
 
         run = VariableExpander.Expand(run, args);
 
+        IReadOnlyDictionary<string, string> artifacts = step is RecipeStep rs
+            ? rs.Artifacts
+            : ReadOnlyDictionary<string, string>.Empty;
+
         ExecutionCommand stepCommand = new ExecutionCommand
         {
             Id = step.Id,
@@ -124,12 +123,12 @@ public class ExecutionPlan
             Command = run,
             Role = step switch
             {
-                ArtifactHandlerStep => CommandRole.ArtifactHandler,
                 RecipeStep => CommandRole.Step,
                 CleanupStep => CommandRole.Cleanup,
                 _ => throw new ArgumentOutOfRangeException()
             },
             FailBehavior = fail.Value,
+            Artifacts = artifacts,
             Deferred = defer
         };