1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
namespace marlin.agents.shell;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using core.util;
using Microsoft.Extensions.FileSystemGlobbing;
using Microsoft.Extensions.FileSystemGlobbing.Abstractions;
public class BlobArtifactStore : IArtifactStore
{
private static readonly string _storeDir = Path.Combine(CommonPaths.AgentRoot, "store");
private static readonly string _blobDir = Path.Combine(_storeDir, "blob");
private static readonly string _manifestFile = Path.Combine(_storeDir, "manifest.json");
private Dictionary<ArtifactManifestKey, ArtifactManifestEntry[]> _manifest
{
get => field != null ? field : throw new InvalidOperationException("Init not run, cannot access manifest");
set;
}
public void Init()
{
Directory.CreateDirectory(_storeDir);
Directory.CreateDirectory(_blobDir);
if (File.Exists(_manifestFile))
{
_manifest = JsonSerializer.Deserialize<Dictionary<ArtifactManifestKey, ArtifactManifestEntry[]>>(
File.ReadAllText(_manifestFile))
?? throw new InvalidOperationException("Could not read manifest, possibly corrupted");
}
else
_manifest = new();
}
public async Task Stow(Guid jobId, string stepId, string artifactId, string basePath, string glob)
{
var key = new ArtifactManifestKey { JobId = jobId, StepId = stepId, ArtifactId = artifactId };
var entries = new List<ArtifactManifestEntry>();
Matcher matcher = new();
matcher.AddInclude(glob);
var globMatches = matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo(basePath)));
foreach (var globMatch in globMatches.Files)
{
var fullPath = Path.Join(basePath, globMatch.Path);
var entry = new ArtifactManifestEntry
{
RelativePath = globMatch.Path,
Hash = ComputeFileHash(fullPath)
};
var targetPath = Path.Join(_blobDir, entry.Hash);
if (!File.Exists(targetPath))
File.Move(fullPath, targetPath);
else
File.Delete(fullPath);
entries.Add(entry);
}
_manifest[key] = entries.ToArray();
await File.WriteAllTextAsync(_manifestFile, JsonSerializer.Serialize(this._manifest));
}
public async Task<IArtifactAccessHandle> Access(Guid jobId, string stepId, string artifactId)
{
throw new NotImplementedException();
}
private static string ComputeFileHash(string filePath)
{
using var stream = File.OpenRead(filePath);
byte[] hash = SHA256.HashData(stream);
return Convert.ToHexStringLower(hash);
}
}
public class BlobArtifactAccessHandle : IArtifactAccessHandle
{
public string Path { get; }
public void Dispose() => Directory.Delete(Path, true);
}
public class ArtifactManifestKey
{
public required Guid JobId { get; init; }
public required string StepId { get; init; }
public required string ArtifactId { get; init; }
public override int GetHashCode() => HashCode.Combine(JobId.GetHashCode(), StepId.GetHashCode(), ArtifactId.GetHashCode());
}
public class ArtifactManifestEntry
{
public required string RelativePath { get; init; }
public required string Hash { get; init; }
}
|
