summaryrefslogtreecommitdiff
path: root/src/marlin.core
diff options
context:
space:
mode:
Diffstat (limited to 'src/marlin.core')
-rw-r--r--src/marlin.core/functional/WorkflowBuilder.cs33
-rw-r--r--src/marlin.core/marlin.core.csproj1
-rw-r--r--src/marlin.core/models/meta/Artifact.cs26
-rw-r--r--src/marlin.core/models/meta/ArtifactHandler.cs12
-rw-r--r--src/marlin.core/models/meta/ArtifactHandlerStep.cs14
-rw-r--r--src/marlin.core/models/meta/RecipeStep.cs3
-rw-r--r--src/marlin.core/models/meta/dto/ArtifactDto.cs8
-rw-r--r--src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs6
-rw-r--r--src/marlin.core/models/meta/dto/StepDto.cs2
-rw-r--r--src/marlin.core/models/meta/dto/WorkflowFileDto.cs1
-rw-r--r--src/marlin.core/models/plan/CommandRole.cs3
-rw-r--r--src/marlin.core/models/plan/ExecutionCommand.cs1
-rw-r--r--src/marlin.core/models/plan/ExecutionPlan.cs15
13 files changed, 14 insertions, 111 deletions
diff --git a/src/marlin.core/functional/WorkflowBuilder.cs b/src/marlin.core/functional/WorkflowBuilder.cs
index 79da909..a912196 100644
--- a/src/marlin.core/functional/WorkflowBuilder.cs
+++ b/src/marlin.core/functional/WorkflowBuilder.cs
@@ -34,7 +34,6 @@ public class WorkflowBuilder
this.ApplyRequirements(file.Requirements);
this.ApplyRecipes(file.Recipes);
this.ApplySteps(file.Steps);
- this.ApplyArtifactHandlers(file.ArtifactHandlers);
}
private void ApplyRequirements(Dictionary<string, RequirementSetDto>? requirements)
@@ -95,19 +94,11 @@ public class WorkflowBuilder
ApplyAbstractStep(step, dto);
step.BaseId = dto.Base;
- step.Ingest = dto.Ingest ?? [];
if (dto.Artifacts != null)
{
- foreach (var (artifactId, artifactDto) in dto.Artifacts)
- {
- step.Artifacts[artifactId] = new Artifact(artifactId)
- {
- Path = artifactDto.Path!,
- Type = artifactDto.As!,
- Handle = artifactDto.Handle ?? ArtifactHandle.Defer
- };
- }
+ foreach (var item in dto.Artifacts)
+ step.Artifacts[item.Key] = item.Value;
}
if (dto.Cleanup != null)
@@ -123,26 +114,6 @@ public class WorkflowBuilder
return step;
}
- private void ApplyArtifactHandlers(Dictionary<string, ArtifactHandlerDto>? handlers)
- {
- if (handlers == null)
- return;
-
- foreach (var (id, dto) in handlers)
- {
- var handler = new ArtifactHandler(id);
-
- if (dto.Handle != null)
- {
- var handleStep = new ArtifactHandlerStep(id + "/handle");
- ApplyAbstractStep(handleStep, dto.Handle);
- handler.Handle = handleStep;
- }
-
- this._workflow.ArtifactHandlers[id] = handler;
- }
- }
-
private static void ApplyAbstractStep(AbstractStep step, StepDtoBase dto)
{
step.Name = dto.Name;
diff --git a/src/marlin.core/marlin.core.csproj b/src/marlin.core/marlin.core.csproj
index 044c698..7a1681c 100644
--- a/src/marlin.core/marlin.core.csproj
+++ b/src/marlin.core/marlin.core.csproj
@@ -7,6 +7,7 @@
</PropertyGroup>
<ItemGroup>
+ <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="10.0.14" />
<PackageReference Include="protobuf-net.Grpc" Version="1.3.14" />
<PackageReference Include="YamlDotNet" Version="18.1.0" />
</ItemGroup>
diff --git a/src/marlin.core/models/meta/Artifact.cs b/src/marlin.core/models/meta/Artifact.cs
deleted file mode 100644
index dce25a6..0000000
--- a/src/marlin.core/models/meta/Artifact.cs
+++ /dev/null
@@ -1,26 +0,0 @@
-namespace marlin.core.models.meta;
-
-public class Artifact
-{
- public Artifact(string id)
- {
- this.Id = id;
- }
-
- public string Id { get; }
- public string Path { get; set; }
- public string Type { get; set; }
- public ArtifactHandle Handle { get; set; } = ArtifactHandle.Defer;
-
- public bool RequiresOrchestratorUpload()
- {
- return this.Type?.StartsWith("test/") == true || this.Type?.StartsWith("logs/") == true;
- }
-}
-
-public enum ArtifactHandle
-{
- Immediate,
- Step,
- Defer
-} \ No newline at end of file
diff --git a/src/marlin.core/models/meta/ArtifactHandler.cs b/src/marlin.core/models/meta/ArtifactHandler.cs
deleted file mode 100644
index 51b6519..0000000
--- a/src/marlin.core/models/meta/ArtifactHandler.cs
+++ /dev/null
@@ -1,12 +0,0 @@
-namespace marlin.core.models.meta;
-
-public class ArtifactHandler
-{
- public ArtifactHandler(string id)
- {
- this.Id = id;
- }
-
- public string Id { get; }
- public ArtifactHandlerStep? Handle { get; set; }
-}
diff --git a/src/marlin.core/models/meta/ArtifactHandlerStep.cs b/src/marlin.core/models/meta/ArtifactHandlerStep.cs
deleted file mode 100644
index bfcc9b9..0000000
--- a/src/marlin.core/models/meta/ArtifactHandlerStep.cs
+++ /dev/null
@@ -1,14 +0,0 @@
-namespace marlin.core.models.meta;
-
-public class ArtifactHandlerStep : AbstractStep
-{
- public ArtifactHandlerStep(string id) : base(id)
- {
- }
-
- public CleanupStep? Cleanup { get; set; } = null;
- public override IEnumerable<AbstractStep> GetAncestorsAndThis(Workflow workflow)
- {
- yield return this;
- }
-}
diff --git a/src/marlin.core/models/meta/RecipeStep.cs b/src/marlin.core/models/meta/RecipeStep.cs
index afe77cf..5cb1f78 100644
--- a/src/marlin.core/models/meta/RecipeStep.cs
+++ b/src/marlin.core/models/meta/RecipeStep.cs
@@ -8,8 +8,7 @@ public class RecipeStep : AbstractStep
public string? BaseId { get; set; }
public CleanupStep? Cleanup { get; set; }
- public Dictionary<string, Artifact> Artifacts { get; set; } = new();
- public List<string> Ingest { get; set; } = new();
+ public Dictionary<string, string> Artifacts { get; set; } = new();
public RecipeStep? GetBase(Workflow workflow) =>
this.BaseId != null ? workflow.Steps[this.BaseId] : null;
diff --git a/src/marlin.core/models/meta/dto/ArtifactDto.cs b/src/marlin.core/models/meta/dto/ArtifactDto.cs
deleted file mode 100644
index aa2db76..0000000
--- a/src/marlin.core/models/meta/dto/ArtifactDto.cs
+++ /dev/null
@@ -1,8 +0,0 @@
-namespace marlin.core.models.meta.dto;
-
-public sealed record ArtifactDto
-{
- public string? Path { get; init; }
- public string? As { get; init; }
- public ArtifactHandle? Handle { get; init; }
-}
diff --git a/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs b/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs
deleted file mode 100644
index 0e51a52..0000000
--- a/src/marlin.core/models/meta/dto/ArtifactHandlerDto.cs
+++ /dev/null
@@ -1,6 +0,0 @@
-namespace marlin.core.models.meta.dto;
-
-public sealed record ArtifactHandlerDto
-{
- public StepDtoBase? Handle { get; init; }
-}
diff --git a/src/marlin.core/models/meta/dto/StepDto.cs b/src/marlin.core/models/meta/dto/StepDto.cs
index 57e4092..c9ffd82 100644
--- a/src/marlin.core/models/meta/dto/StepDto.cs
+++ b/src/marlin.core/models/meta/dto/StepDto.cs
@@ -4,6 +4,6 @@ public sealed record StepDto : StepDtoBase
{
public string? Base { get; init; }
public CleanupStepDto? Cleanup { get; init; }
- public Dictionary<string, ArtifactDto>? Artifacts { get; init; }
+ public Dictionary<string, string>? Artifacts { get; init; }
public List<string>? Ingest { get; init; }
}
diff --git a/src/marlin.core/models/meta/dto/WorkflowFileDto.cs b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs
index 10997d6..2a8d626 100644
--- a/src/marlin.core/models/meta/dto/WorkflowFileDto.cs
+++ b/src/marlin.core/models/meta/dto/WorkflowFileDto.cs
@@ -5,5 +5,4 @@ public sealed record WorkflowFileDto
public Dictionary<string, RecipeDto>? Recipes { get; init; }
public Dictionary<string, StepDto>? Steps { get; init; }
public Dictionary<string, RequirementSetDto>? Requirements { get; init; }
- public Dictionary<string, ArtifactHandlerDto>? ArtifactHandlers { get; init; }
}
diff --git a/src/marlin.core/models/plan/CommandRole.cs b/src/marlin.core/models/plan/CommandRole.cs
index 8005532..f47b51e 100644
--- a/src/marlin.core/models/plan/CommandRole.cs
+++ b/src/marlin.core/models/plan/CommandRole.cs
@@ -3,6 +3,5 @@ namespace marlin.core.models.plan;
public enum CommandRole
{
Step = 0,
- ArtifactHandler = 1,
- Cleanup = 2
+ Cleanup = 1
} \ No newline at end of file
diff --git a/src/marlin.core/models/plan/ExecutionCommand.cs b/src/marlin.core/models/plan/ExecutionCommand.cs
index 4bf19c3..9649eec 100644
--- a/src/marlin.core/models/plan/ExecutionCommand.cs
+++ b/src/marlin.core/models/plan/ExecutionCommand.cs
@@ -9,6 +9,7 @@ public class ExecutionCommand
public required string Command;
public required CommandRole Role;
public required bool Deferred;
+ public required IReadOnlyDictionary<string, string> Artifacts;
public CommandResultSelector? ResultSelector;
public required FailBehavior FailBehavior;
diff --git a/src/marlin.core/models/plan/ExecutionPlan.cs b/src/marlin.core/models/plan/ExecutionPlan.cs
index 24c40e4..fa84634 100644
--- a/src/marlin.core/models/plan/ExecutionPlan.cs
+++ b/src/marlin.core/models/plan/ExecutionPlan.cs
@@ -1,20 +1,19 @@
namespace marlin.core.models.plan;
+using System.Collections.ObjectModel;
using meta;
using util;
public class ExecutionPlan
{
- public IEnumerable<ExecutionCommand> Components => [..this._commands, ..this._deferredArtifactHandler, ..this._deferredCleanup];
+ public IEnumerable<ExecutionCommand> Components => [..this._commands, ..this._deferredCleanup];
private readonly List<ExecutionCommand> _commands;
- private readonly List<ExecutionCommand> _deferredArtifactHandler;
private readonly List<ExecutionCommand> _deferredCleanup;
public ExecutionPlan()
{
this._commands = new();
- this._deferredArtifactHandler = new();
this._deferredCleanup = new();
}
@@ -31,10 +30,6 @@ public class ExecutionPlan
this._deferredCleanup.Add(command);
break;
- case CommandRole.ArtifactHandler:
- this._deferredArtifactHandler.Add(command);
- break;
-
default:
throw new InvalidOperationException($"Cannot defer {command.Role}");
}
@@ -117,6 +112,10 @@ public class ExecutionPlan
run = VariableExpander.Expand(run, args);
+ IReadOnlyDictionary<string, string> artifacts = step is RecipeStep rs
+ ? rs.Artifacts
+ : ReadOnlyDictionary<string, string>.Empty;
+
ExecutionCommand stepCommand = new ExecutionCommand
{
Id = step.Id,
@@ -124,12 +123,12 @@ public class ExecutionPlan
Command = run,
Role = step switch
{
- ArtifactHandlerStep => CommandRole.ArtifactHandler,
RecipeStep => CommandRole.Step,
CleanupStep => CommandRole.Cleanup,
_ => throw new ArgumentOutOfRangeException()
},
FailBehavior = fail.Value,
+ Artifacts = artifacts,
Deferred = defer
};
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-31 15:35:20 +0000