summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/marlin.agents.shell/BlobArtifactStore.cs102
-rw-r--r--src/marlin.agents.shell/IArtifactStore.cs15
-rw-r--r--src/marlin.agents.shell/LocalInstanceService.cs13
-rw-r--r--src/marlin.agents.shell/Program.cs13
-rw-r--r--src/marlin.core/functional/WorkflowBuilder.cs33
-rw-r--r--src/marlin.core/marlin.core.csproj1
-rw-r--r--src/marlin.core/models/meta/Artifact.cs26
-rw-r--r--src/marlin.core/models/meta/ArtifactHandler.cs12
-rw-r--r--src/marlin.core/models/meta/ArtifactHandlerStep.cs14
-rw-r--r--src/marlin.core/models/meta/RecipeStep.cs3
-rw-r--r--src/marlin.core/models/meta/dto/ArtifactDto.cs8
-rw-r--r--src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs6
-rw-r--r--src/marlin.core/models/meta/dto/StepDto.cs2
-rw-r--r--src/marlin.core/models/meta/dto/WorkflowFileDto.cs1
-rw-r--r--src/marlin.core/models/plan/CommandRole.cs3
-rw-r--r--src/marlin.core/models/plan/ExecutionCommand.cs1
-rw-r--r--src/marlin.core/models/plan/ExecutionPlan.cs15
17 files changed, 144 insertions, 124 deletions
diff --git a/src/marlin.agents.shell/BlobArtifactStore.cs b/src/marlin.agents.shell/BlobArtifactStore.cs
new file mode 100644
index 0000000..e447c5a
--- /dev/null
+++ b/src/marlin.agents.shell/BlobArtifactStore.cs
@@ -0,0 +1,102 @@
+namespace marlin.agents.shell;
+
+using System.Security.Cryptography;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using core.util;
+using Microsoft.Extensions.FileSystemGlobbing;
+using Microsoft.Extensions.FileSystemGlobbing.Abstractions;
+
+public class BlobArtifactStore : IArtifactStore
+{
+ private static readonly string _storeDir = Path.Combine(CommonPaths.AgentRoot, "store");
+ private static readonly string _blobDir = Path.Combine(_storeDir, "blob");
+ private static readonly string _manifestFile = Path.Combine(_storeDir, "manifest.json");
+
+ private Dictionary<ArtifactManifestKey, ArtifactManifestEntry[]> _manifest
+ {
+ get => field != null ? field : throw new InvalidOperationException("Init not run, cannot access manifest");
+ set;
+ }
+
+ public void Init()
+ {
+ Directory.CreateDirectory(_storeDir);
+ Directory.CreateDirectory(_blobDir);
+
+ if (File.Exists(_manifestFile))
+ {
+ _manifest = JsonSerializer.Deserialize<Dictionary<ArtifactManifestKey, ArtifactManifestEntry[]>>(
+ File.ReadAllText(_manifestFile))
+ ?? throw new InvalidOperationException("Could not read manifest, possibly corrupted");
+ }
+ else
+ _manifest = new();
+ }
+
+ public async Task Stow(Guid jobId, string stepId, string artifactId, string basePath, string glob)
+ {
+ var key = new ArtifactManifestKey { JobId = jobId, StepId = stepId, ArtifactId = artifactId };
+ var entries = new List<ArtifactManifestEntry>();
+
+ Matcher matcher = new();
+ matcher.AddInclude(glob);
+ var globMatches = matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo(basePath)));
+
+ foreach (var globMatch in globMatches.Files)
+ {
+ var fullPath = Path.Join(basePath, globMatch.Path);
+ var entry = new ArtifactManifestEntry
+ {
+ RelativePath = globMatch.Path,
+ Hash = ComputeFileHash(fullPath)
+ };
+
+ var targetPath = Path.Join(_blobDir, entry.Hash);
+
+ if (!File.Exists(targetPath))
+ File.Move(fullPath, targetPath);
+ else
+ File.Delete(fullPath);
+
+ entries.Add(entry);
+ }
+
+ _manifest[key] = entries.ToArray();
+ await File.WriteAllTextAsync(_manifestFile, JsonSerializer.Serialize(this._manifest));
+ }
+
+ public async Task<IArtifactAccessHandle> Access(Guid jobId, string stepId, string artifactId)
+ {
+ throw new NotImplementedException();
+ }
+
+ private static string ComputeFileHash(string filePath)
+ {
+ using var stream = File.OpenRead(filePath);
+ byte[] hash = SHA256.HashData(stream);
+ return Convert.ToHexStringLower(hash);
+ }
+}
+
+public class BlobArtifactAccessHandle : IArtifactAccessHandle
+{
+ public string Path { get; }
+
+ public void Dispose() => Directory.Delete(Path, true);
+}
+
+public class ArtifactManifestKey
+{
+ public required Guid JobId { get; init; }
+ public required string StepId { get; init; }
+ public required string ArtifactId { get; init; }
+
+ public override int GetHashCode() => HashCode.Combine(JobId.GetHashCode(), StepId.GetHashCode(), ArtifactId.GetHashCode());
+}
+
+public class ArtifactManifestEntry
+{
+ public required string RelativePath { get; init; }
+ public required string Hash { get; init; }
+} \ No newline at end of file
diff --git a/src/marlin.agents.shell/IArtifactStore.cs b/src/marlin.agents.shell/IArtifactStore.cs
new file mode 100644
index 0000000..2d6338c
--- /dev/null
+++ b/src/marlin.agents.shell/IArtifactStore.cs
@@ -0,0 +1,15 @@
+namespace marlin.agents.shell;
+
+public interface IArtifactStore
+{
+ /// Extracts given artifacts into shared flat blob storage
+ Task Stow(Guid jobId, string stepId, string artifactId, string basePath, string glob);
+
+ /// Constructs virtual directory structure to access a given artifact
+ Task<IArtifactAccessHandle> Access(Guid jobId, string stepId, string artifactId);
+}
+
+public interface IArtifactAccessHandle : IDisposable
+{
+ public string Path { get; }
+} \ No newline at end of file
diff --git a/src/marlin.agents.shell/LocalInstanceService.cs b/src/marlin.agents.shell/LocalInstanceService.cs
new file mode 100644
index 0000000..d0d2521
--- /dev/null
+++ b/src/marlin.agents.shell/LocalInstanceService.cs
@@ -0,0 +1,13 @@
+using marlin.core.contracts;
+using ProtoBuf.Grpc;
+
+public class LocalInstanceService : IInstanceService
+{
+ public async Task SendLogEvent(LogEventRequest request, CallContext context = default)
+ {
+ TextWriter writer = request.Error ? Console.Error : Console.Out;
+ await writer.WriteLineAsync(request.Message);
+ }
+
+ public Task SendStatusUpdate(StatusUpdateRequest request, CallContext context = default) => Task.CompletedTask;
+} \ No newline at end of file
diff --git a/src/marlin.agents.shell/Program.cs b/src/marlin.agents.shell/Program.cs
index c895e8d..1a3e97f 100644
--- a/src/marlin.agents.shell/Program.cs
+++ b/src/marlin.agents.shell/Program.cs
@@ -2,13 +2,11 @@ using System.Runtime.InteropServices;
using System.Threading.Channels;
using Grpc.Net.Client;
using marlin.agents.shell;
-using marlin.core.contracts;
using marlin.core.functional;
using marlin.core.models;
using marlin.core.models.meta;
using marlin.core.models.plan;
using marlin.core.util;
-using ProtoBuf.Grpc;
using ProtoBuf.Grpc.Client;
DotNetEnv.Env.NoClobber().TraversePath().Load();
@@ -25,15 +23,4 @@ else
JobUnitOfWork uow = new(new LocalInstanceService(), repoPath, recipe);
await uow.Run();
-}
-
-public class LocalInstanceService : IInstanceService
-{
- public async Task SendLogEvent(LogEventRequest request, CallContext context = default)
- {
- TextWriter writer = request.Error ? Console.Error : Console.Out;
- await writer.WriteLineAsync(request.Message);
- }
-
- public Task SendStatusUpdate(StatusUpdateRequest request, CallContext context = default) => Task.CompletedTask;
} \ No newline at end of file
diff --git a/src/marlin.core/functional/WorkflowBuilder.cs b/src/marlin.core/functional/WorkflowBuilder.cs
index 79da909..a912196 100644
--- a/src/marlin.core/functional/WorkflowBuilder.cs
+++ b/src/marlin.core/functional/WorkflowBuilder.cs
@@ -34,7 +34,6 @@ public class WorkflowBuilder
this.ApplyRequirements(file.Requirements);
this.ApplyRecipes(file.Recipes);
this.ApplySteps(file.Steps);
- this.ApplyArtifactHandlers(file.ArtifactHandlers);
}
private void ApplyRequirements(Dictionary<string, RequirementSetDto>? requirements)
@@ -95,19 +94,11 @@ public class WorkflowBuilder
ApplyAbstractStep(step, dto);
step.BaseId = dto.Base;
- step.Ingest = dto.Ingest ?? [];
if (dto.Artifacts != null)
{
- foreach (var (artifactId, artifactDto) in dto.Artifacts)
- {
- step.Artifacts[artifactId] = new Artifact(artifactId)
- {
- Path = artifactDto.Path!,
- Type = artifactDto.As!,
- Handle = artifactDto.Handle ?? ArtifactHandle.Defer
- };
- }
+ foreach (var item in dto.Artifacts)
+ step.Artifacts[item.Key] = item.Value;
}
if (dto.Cleanup != null)
@@ -123,26 +114,6 @@ public class WorkflowBuilder
return step;
}
- private void ApplyArtifactHandlers(Dictionary<string, ArtifactHandlerDto>? handlers)
- {
- if (handlers == null)
- return;
-
- foreach (var (id, dto) in handlers)
- {
- var handler = new ArtifactHandler(id);
-
- if (dto.Handle != null)
- {
- var handleStep = new ArtifactHandlerStep(id + "/handle");
- ApplyAbstractStep(handleStep, dto.Handle);
- handler.Handle = handleStep;
- }
-
- this._workflow.ArtifactHandlers[id] = handler;
- }
- }
-
private static void ApplyAbstractStep(AbstractStep step, StepDtoBase dto)
{
step.Name = dto.Name;
diff --git a/src/marlin.core/marlin.core.csproj b/src/marlin.core/marlin.core.csproj
index 044c698..7a1681c 100644
--- a/src/marlin.core/marlin.core.csproj
+++ b/src/marlin.core/marlin.core.csproj
@@ -7,6 +7,7 @@
</PropertyGroup>
<ItemGroup>
+ <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="10.0.14" />
<PackageReference Include="protobuf-net.Grpc" Version="1.3.14" />
<PackageReference Include="YamlDotNet" Version="18.1.0" />
</ItemGroup>
diff --git a/src/marlin.core/models/meta/Artifact.cs b/src/marlin.core/models/meta/Artifact.cs
deleted file mode 100644
index dce25a6..0000000
--- a/src/marlin.core/models/meta/Artifact.cs
+++ /dev/null
@@ -1,26 +0,0 @@
-namespace marlin.core.models.meta;
-
-public class Artifact
-{
- public Artifact(string id)
- {
- this.Id = id;
- }
-
- public string Id { get; }
- public string Path { get; set; }
- public string Type { get; set; }
- public ArtifactHandle Handle { get; set; } = ArtifactHandle.Defer;
-
- public bool RequiresOrchestratorUpload()
- {
- return this.Type?.StartsWith("test/") == true || this.Type?.StartsWith("logs/") == true;
- }
-}
-
-public enum ArtifactHandle
-{
- Immediate,
- Step,
- Defer
-} \ No newline at end of file
diff --git a/src/marlin.core/models/meta/ArtifactHandler.cs b/src/marlin.core/models/meta/ArtifactHandler.cs
deleted file mode 100644
index 51b6519..0000000
--- a/src/marlin.core/models/meta/ArtifactHandler.cs
+++ /dev/null
@@ -1,12 +0,0 @@
-namespace marlin.core.models.meta;
-
-public class ArtifactHandler
-{
- public ArtifactHandler(string id)
- {
- this.Id = id;
- }
-
- public string Id { get; }
- public ArtifactHandlerStep? Handle { get; set; }
-}
diff --git a/src/marlin.core/models/meta/ArtifactHandlerStep.cs b/src/marlin.core/models/meta/ArtifactHandlerStep.cs
deleted file mode 100644
index bfcc9b9..0000000
--- a/src/marlin.core/models/meta/ArtifactHandlerStep.cs
+++ /dev/null
@@ -1,14 +0,0 @@
-namespace marlin.core.models.meta;
-
-public class ArtifactHandlerStep : AbstractStep
-{
- public ArtifactHandlerStep(string id) : base(id)
- {
- }
-
- public CleanupStep? Cleanup { get; set; } = null;
- public override IEnumerable<AbstractStep> GetAncestorsAndThis(Workflow workflow)
- {
- yield return this;
- }
-}
diff --git a/src/marlin.core/models/meta/RecipeStep.cs b/src/marlin.core/models/meta/RecipeStep.cs
index afe77cf..5cb1f78 100644
--- a/src/marlin.core/models/meta/RecipeStep.cs
+++ b/src/marlin.core/models/meta/RecipeStep.cs
@@ -8,8 +8,7 @@ public class RecipeStep : AbstractStep
public string? BaseId { get; set; }
public CleanupStep? Cleanup { get; set; }
- public Dictionary<string, Artifact> Artifacts { get; set; } = new();
- public List<string> Ingest { get; set; } = new();
+ public Dictionary<string, string> Artifacts { get; set; } = new();
public RecipeStep? GetBase(Workflow workflow) =>
this.BaseId != null ? workflow.Steps[this.BaseId] : null;
diff --git a/src/marlin.core/models/meta/dto/ArtifactDto.cs b/src/marlin.core/models/meta/dto/ArtifactDto.cs
deleted file mode 100644
index aa2db76..0000000
--- a/src/marlin.core/models/meta/dto/ArtifactDto.cs
+++ /dev/null
@@ -1,8 +0,0 @@
-namespace marlin.core.models.meta.dto;
-
-public sealed record ArtifactDto
-{
- public string? Path { get; init; }
- public string? As { get; init; }
- public ArtifactHandle? Handle { get; init; }
-}
diff --git a/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs b/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs
deleted file mode 100644
index 0e51a52..0000000
--- a/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs
+++ /dev/null
@@ -1,6 +0,0 @@
-namespace marlin.core.models.meta.dto;
-
-public sealed record ArtifactHandlerDto
-{
- public StepDtoBase? Handle { get; init; }
-}
diff --git a/src/marlin.core/models/meta/dto/StepDto.cs b/src/marlin.core/models/meta/dto/StepDto.cs
index 57e4092..c9ffd82 100644
--- a/src/marlin.core/models/meta/dto/StepDto.cs
+++ b/src/marlin.core/models/meta/dto/StepDto.cs
@@ -4,6 +4,6 @@ public sealed record StepDto : StepDtoBase
{
public string? Base { get; init; }
public CleanupStepDto? Cleanup { get; init; }
- public Dictionary<string, ArtifactDto>? Artifacts { get; init; }
+ public Dictionary<string, string>? Artifacts { get; init; }
public List<string>? Ingest { get; init; }
}
diff --git a/src/marlin.core/models/meta/dto/WorkflowFileDto.cs b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs
index 10997d6..2a8d626 100644
--- a/src/marlin.core/models/meta/dto/WorkflowFileDto.cs
+++ b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs
@@ -5,5 +5,4 @@ public sealed record WorkflowFileDto
public Dictionary<string, RecipeDto>? Recipes { get; init; }
public Dictionary<string, StepDto>? Steps { get; init; }
public Dictionary<string, RequirementSetDto>? Requirements { get; init; }
- public Dictionary<string, ArtifactHandlerDto>? ArtifactHandlers { get; init; }
}
diff --git a/src/marlin.core/models/plan/CommandRole.cs b/src/marlin.core/models/plan/CommandRole.cs
index 8005532..f47b51e 100644
--- a/src/marlin.core/models/plan/CommandRole.cs
+++ b/src/marlin.core/models/plan/CommandRole.cs
@@ -3,6 +3,5 @@ namespace marlin.core.models.plan;
public enum CommandRole
{
Step = 0,
- ArtifactHandler = 1,
- Cleanup = 2
+ Cleanup = 1
} \ No newline at end of file
diff --git a/src/marlin.core/models/plan/ExecutionCommand.cs b/src/marlin.core/models/plan/ExecutionCommand.cs
index 4bf19c3..9649eec 100644
--- a/src/marlin.core/models/plan/ExecutionCommand.cs
+++ b/src/marlin.core/models/plan/ExecutionCommand.cs
@@ -9,6 +9,7 @@ public class ExecutionCommand
public required string Command;
public required CommandRole Role;
public required bool Deferred;
+ public required IReadOnlyDictionary<string, string> Artifacts;
public CommandResultSelector? ResultSelector;
public required FailBehavior FailBehavior;
diff --git a/src/marlin.core/models/plan/ExecutionPlan.cs b/src/marlin.core/models/plan/ExecutionPlan.cs
index 24c40e4..fa84634 100644
--- a/src/marlin.core/models/plan/ExecutionPlan.cs
+++ b/src/marlin.core/models/plan/ExecutionPlan.cs
@@ -1,20 +1,19 @@
namespace marlin.core.models.plan;
+using System.Collections.ObjectModel;
using meta;
using util;
public class ExecutionPlan
{
- public IEnumerable<ExecutionCommand> Components => [..this._commands, ..this._deferredArtifactHandler, ..this._deferredCleanup];
+ public IEnumerable<ExecutionCommand> Components => [..this._commands, ..this._deferredCleanup];
private readonly List<ExecutionCommand> _commands;
- private readonly List<ExecutionCommand> _deferredArtifactHandler;
private readonly List<ExecutionCommand> _deferredCleanup;
public ExecutionPlan()
{
this._commands = new();
- this._deferredArtifactHandler = new();
this._deferredCleanup = new();
}
@@ -31,10 +30,6 @@ public class ExecutionPlan
this._deferredCleanup.Add(command);
break;
- case CommandRole.ArtifactHandler:
- this._deferredArtifactHandler.Add(command);
- break;
-
default:
throw new InvalidOperationException($"Cannot defer {command.Role}");
}
@@ -117,6 +112,10 @@ public class ExecutionPlan
run = VariableExpander.Expand(run, args);
+ IReadOnlyDictionary<string, string> artifacts = step is RecipeStep rs
+ ? rs.Artifacts
+ : ReadOnlyDictionary<string, string>.Empty;
+
ExecutionCommand stepCommand = new ExecutionCommand
{
Id = step.Id,
@@ -124,12 +123,12 @@ public class ExecutionPlan
Command = run,
Role = step switch
{
- ArtifactHandlerStep => CommandRole.ArtifactHandler,
RecipeStep => CommandRole.Step,
CleanupStep => CommandRole.Cleanup,
_ => throw new ArgumentOutOfRangeException()
},
FailBehavior = fail.Value,
+ Artifacts = artifacts,
Deferred = defer
};
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-31 20:11:05 +0000