From 1e892b09b46de9908ddbdc895031895101313a26 Mon Sep 17 00:00:00 2001 From: Melodi <62381002+Melodi17@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:40:13 +1000 Subject: refactor: initial artifact cleanup and implementation --- src/marlin.agents.shell/BlobArtifactStore.cs | 102 ++++++++++++++++++++++++ src/marlin.agents.shell/IArtifactStore.cs | 15 ++++ src/marlin.agents.shell/LocalInstanceService.cs | 13 +++ src/marlin.agents.shell/Program.cs | 13 --- 4 files changed, 130 insertions(+), 13 deletions(-) create mode 100644 src/marlin.agents.shell/BlobArtifactStore.cs create mode 100644 src/marlin.agents.shell/IArtifactStore.cs create mode 100644 src/marlin.agents.shell/LocalInstanceService.cs (limited to 'src/marlin.agents.shell') diff --git a/src/marlin.agents.shell/BlobArtifactStore.cs b/src/marlin.agents.shell/BlobArtifactStore.cs new file mode 100644 index 0000000..e447c5a --- /dev/null +++ b/src/marlin.agents.shell/BlobArtifactStore.cs @@ -0,0 +1,102 @@ +namespace marlin.agents.shell; + +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using core.util; +using Microsoft.Extensions.FileSystemGlobbing; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; + +public class BlobArtifactStore : IArtifactStore +{ + private static readonly string _storeDir = Path.Combine(CommonPaths.AgentRoot, "store"); + private static readonly string _blobDir = Path.Combine(_storeDir, "blob"); + private static readonly string _manifestFile = Path.Combine(_storeDir, "manifest.json"); + + private Dictionary _manifest + { + get => field != null ? field : throw new InvalidOperationException("Init not run, cannot access manifest"); + set; + } + + public void Init() + { + Directory.CreateDirectory(_storeDir); + Directory.CreateDirectory(_blobDir); + + if (File.Exists(_manifestFile)) + { + _manifest = JsonSerializer.Deserialize>( + File.ReadAllText(_manifestFile)) + ?? throw new InvalidOperationException("Could not read manifest, possibly corrupted"); + } + else + _manifest = new(); + } + + public async Task Stow(Guid jobId, string stepId, string artifactId, string basePath, string glob) + { + var key = new ArtifactManifestKey { JobId = jobId, StepId = stepId, ArtifactId = artifactId }; + var entries = new List(); + + Matcher matcher = new(); + matcher.AddInclude(glob); + var globMatches = matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo(basePath))); + + foreach (var globMatch in globMatches.Files) + { + var fullPath = Path.Join(basePath, globMatch.Path); + var entry = new ArtifactManifestEntry + { + RelativePath = globMatch.Path, + Hash = ComputeFileHash(fullPath) + }; + + var targetPath = Path.Join(_blobDir, entry.Hash); + + if (!File.Exists(targetPath)) + File.Move(fullPath, targetPath); + else + File.Delete(fullPath); + + entries.Add(entry); + } + + _manifest[key] = entries.ToArray(); + await File.WriteAllTextAsync(_manifestFile, JsonSerializer.Serialize(this._manifest)); + } + + public async Task Access(Guid jobId, string stepId, string artifactId) + { + throw new NotImplementedException(); + } + + private static string ComputeFileHash(string filePath) + { + using var stream = File.OpenRead(filePath); + byte[] hash = SHA256.HashData(stream); + return Convert.ToHexStringLower(hash); + } +} + +public class BlobArtifactAccessHandle : IArtifactAccessHandle +{ + public string Path { get; } + + public void Dispose() => Directory.Delete(Path, true); +} + +public class ArtifactManifestKey +{ + public required Guid JobId { get; init; } + public required string StepId { get; init; } + public required string ArtifactId { get; init; } + + public override int GetHashCode() => HashCode.Combine(JobId.GetHashCode(), StepId.GetHashCode(), ArtifactId.GetHashCode()); +} + +public class ArtifactManifestEntry +{ + public required string RelativePath { get; init; } + public required string Hash { get; init; } +} \ No newline at end of file diff --git a/src/marlin.agents.shell/IArtifactStore.cs b/src/marlin.agents.shell/IArtifactStore.cs new file mode 100644 index 0000000..2d6338c --- /dev/null +++ b/src/marlin.agents.shell/IArtifactStore.cs @@ -0,0 +1,15 @@ +namespace marlin.agents.shell; + +public interface IArtifactStore +{ + /// Extracts given artifacts into shared flat blob storage + Task Stow(Guid jobId, string stepId, string artifactId, string basePath, string glob); + + /// Constructs virtual directory structure to access a given artifact + Task Access(Guid jobId, string stepId, string artifactId); +} + +public interface IArtifactAccessHandle : IDisposable +{ + public string Path { get; } +} \ No newline at end of file diff --git a/src/marlin.agents.shell/LocalInstanceService.cs b/src/marlin.agents.shell/LocalInstanceService.cs new file mode 100644 index 0000000..d0d2521 --- /dev/null +++ b/src/marlin.agents.shell/LocalInstanceService.cs @@ -0,0 +1,13 @@ +using marlin.core.contracts; +using ProtoBuf.Grpc; + +public class LocalInstanceService : IInstanceService +{ + public async Task SendLogEvent(LogEventRequest request, CallContext context = default) + { + TextWriter writer = request.Error ? Console.Error : Console.Out; + await writer.WriteLineAsync(request.Message); + } + + public Task SendStatusUpdate(StatusUpdateRequest request, CallContext context = default) => Task.CompletedTask; +} \ No newline at end of file diff --git a/src/marlin.agents.shell/Program.cs b/src/marlin.agents.shell/Program.cs index c895e8d..1a3e97f 100644 --- a/src/marlin.agents.shell/Program.cs +++ b/src/marlin.agents.shell/Program.cs @@ -2,13 +2,11 @@ using System.Runtime.InteropServices; using System.Threading.Channels; using Grpc.Net.Client; using marlin.agents.shell; -using marlin.core.contracts; using marlin.core.functional; using marlin.core.models; using marlin.core.models.meta; using marlin.core.models.plan; using marlin.core.util; -using ProtoBuf.Grpc; using ProtoBuf.Grpc.Client; DotNetEnv.Env.NoClobber().TraversePath().Load(); @@ -25,15 +23,4 @@ else JobUnitOfWork uow = new(new LocalInstanceService(), repoPath, recipe); await uow.Run(); -} - -public class LocalInstanceService : IInstanceService -{ - public async Task SendLogEvent(LogEventRequest request, CallContext context = default) - { - TextWriter writer = request.Error ? Console.Error : Console.Out; - await writer.WriteLineAsync(request.Message); - } - - public Task SendStatusUpdate(StatusUpdateRequest request, CallContext context = default) => Task.CompletedTask; } \ No newline at end of file -- cgit v1.3.1