Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions src/Netclaw.SkillClient/Models.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,20 @@ public sealed record RfcResourceEntry

[JsonPropertyName("url")]
public string Url { get; init; } = "";

[JsonPropertyName("unixMode")]
public int? UnixMode { get; init; }

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have to preserve the unix mode on uploaded files if we're including executable resources

}

public sealed record SkillResourceUpload(string RelativePath, Stream Content, int? UnixMode = null);

public sealed record SkillResourceUploadMetadata
{
[JsonPropertyName("path")]
public string Path { get; init; } = "";

[JsonPropertyName("unixMode")]
public int? UnixMode { get; init; }
}

/// <summary>
Expand Down Expand Up @@ -244,6 +258,7 @@ public sealed record ErrorResponse
[JsonSerializable(typeof(IReadOnlyList<ApiKeySummary>))]
[JsonSerializable(typeof(IReadOnlyList<CheckUpdateRequest>))]
[JsonSerializable(typeof(IReadOnlyList<CheckUpdateResponse>))]
[JsonSerializable(typeof(IReadOnlyList<SkillResourceUploadMetadata>))]
[JsonSerializable(typeof(CreateApiKeyRequest))]
[JsonSerializable(typeof(ErrorResponse))]
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)]
Expand Down
52 changes: 48 additions & 4 deletions src/Netclaw.SkillClient/SkillServerClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -194,13 +194,27 @@ public async Task<SkillUploadResponse> UploadSkillAsync(
string name, string version, Stream skillMdContent, string? category = null,
CancellationToken ct = default)
{
return await UploadSkillWithResourcesAsync(name, version, skillMdContent, [], category, ct);
return await UploadSkillWithResourceUploadsAsync(name, version, skillMdContent, Array.Empty<SkillResourceUpload>(), category, ct);
}

public async Task<SkillUploadResponse> UploadSkillWithResourcesAsync(
string name, string version, Stream skillMdContent,
IReadOnlyList<(string RelativePath, Stream Content)> resources,
string? category = null, CancellationToken ct = default)
{
return await UploadSkillWithResourceUploadsAsync(
name,
version,
skillMdContent,
resources.Select(r => new SkillResourceUpload(r.RelativePath, r.Content)).ToList(),
category,
ct);
}

public async Task<SkillUploadResponse> UploadSkillWithResourceUploadsAsync(
string name, string version, Stream skillMdContent,
IReadOnlyList<SkillResourceUpload> resources,
string? category = null, CancellationToken ct = default)
{
using var response = await PostSkillAsync(name, version, skillMdContent, resources, category, ct);
response.EnsureSuccessStatusCode();
Expand All @@ -212,6 +226,20 @@ public async Task<SkillUploadResponse> UploadSkillWithResourcesAsync(
string name, string version, Stream skillMdContent,
IReadOnlyList<(string RelativePath, Stream Content)> resources,
string? category = null, CancellationToken ct = default)
{
return await UploadSkillIfNotExistsWithResourceUploadsAsync(
name,
version,
skillMdContent,
resources.Select(r => new SkillResourceUpload(r.RelativePath, r.Content)).ToList(),
category,
ct);
}

public async Task<SkillUploadResponse?> UploadSkillIfNotExistsWithResourceUploadsAsync(
string name, string version, Stream skillMdContent,
IReadOnlyList<SkillResourceUpload> resources,
string? category = null, CancellationToken ct = default)
{
using var response = await PostSkillAsync(name, version, skillMdContent, resources, category, ct);

Expand All @@ -225,7 +253,7 @@ public async Task<SkillUploadResponse> UploadSkillWithResourcesAsync(

private async Task<HttpResponseMessage> PostSkillAsync(
string name, string version, Stream skillMdContent,
IReadOnlyList<(string RelativePath, Stream Content)> resources,
IReadOnlyList<SkillResourceUpload> resources,
string? category, CancellationToken ct)
{
using var content = new MultipartFormDataContent();
Expand All @@ -235,8 +263,24 @@ private async Task<HttpResponseMessage> PostSkillAsync(
content.Add(new StringContent(category), "category");
content.Add(new StreamContent(skillMdContent), "file", "SKILL.md");

foreach (var (relativePath, resourceStream) in resources)
content.Add(new StreamContent(resourceStream), "resources", relativePath);
foreach (var resource in resources)
content.Add(new StreamContent(resource.Content), "resources", resource.RelativePath);

if (resources.Any(resource => resource.UnixMode.HasValue))
{
var metadata = resources
.Where(resource => resource.UnixMode.HasValue)
.Select(resource => new SkillResourceUploadMetadata
{
Path = resource.RelativePath,
UnixMode = resource.UnixMode
})
.ToList();
var json = JsonSerializer.Serialize(
metadata,
SkillServerClientJsonContext.Default.IReadOnlyListSkillResourceUploadMetadata);
content.Add(new StringContent(json), "resourceMetadata");
}

return await _httpClient.PostAsync("skills", content, ct);
}
Expand Down
17 changes: 10 additions & 7 deletions src/Netclaw.SkillServer.Cli/Publishing/PublishOrchestrator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -67,25 +67,28 @@ public async Task<PublishResult> PublishAsync(
}
}

var resources = new List<(string RelativePath, Stream Content)>();
var resources = new List<SkillResourceUpload>();
FileStream? skillMdStream = null;
try
{
skillMdStream = File.OpenRead(skill.SkillMdPath);

foreach (var resource in skill.Resources)
{
resources.Add((resource.RelativePath, File.OpenRead(resource.AbsolutePath)));
resources.Add(new SkillResourceUpload(
resource.RelativePath,
File.OpenRead(resource.AbsolutePath),
resource.UnixMode));
}

if (options.Verbose)
{
ConsoleOutput.WriteDim($" Uploading SKILL.md ({FormatSize(skillMdStream.Length)})");
foreach (var (relativePath, stream) in resources)
ConsoleOutput.WriteDim($" Uploading {relativePath} ({FormatSize(stream.Length)})");
foreach (var resource in resources)
ConsoleOutput.WriteDim($" Uploading {resource.RelativePath} ({FormatSize(resource.Content.Length)})");
}

var uploadResponse = await _client.UploadSkillIfNotExistsAsync(
var uploadResponse = await _client.UploadSkillIfNotExistsWithResourceUploadsAsync(
skill.Name, version, skillMdStream, resources, skill.Category, ct);

if (uploadResponse is null)
Expand All @@ -106,8 +109,8 @@ public async Task<PublishResult> PublishAsync(
{
if (skillMdStream is not null)
await skillMdStream.DisposeAsync();
foreach (var (_, stream) in resources)
await stream.DisposeAsync();
foreach (var resource in resources)
await resource.Content.DisposeAsync();
}
}

Expand Down
49 changes: 46 additions & 3 deletions src/Netclaw.SkillServer.Cli/Publishing/SkillDirectoryScanner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,20 @@ internal sealed record ScannedSkill(
string SkillMdPath,
IReadOnlyList<ScannedResource> Resources);

internal readonly record struct ScannedResource(string RelativePath, string AbsolutePath);
internal readonly record struct ScannedResource(string RelativePath, string AbsolutePath, int? UnixMode);

internal static partial class SkillDirectoryScanner
{
private const UnixFileMode PermissionMask = UnixFileMode.UserRead
| UnixFileMode.UserWrite
| UnixFileMode.UserExecute
| UnixFileMode.GroupRead
| UnixFileMode.GroupWrite
| UnixFileMode.GroupExecute
| UnixFileMode.OtherRead
| UnixFileMode.OtherWrite
| UnixFileMode.OtherExecute;

[GeneratedRegex(@"^---\s*\n(.*?)\n---", RegexOptions.Singleline)]
private static partial Regex FrontmatterRegex();

Expand Down Expand Up @@ -75,17 +85,50 @@ private static IReadOnlyList<ScannedResource> ScanResources(string skillDirector

foreach (var subDir in Directory.EnumerateDirectories(skillDirectory))
{
foreach (var file in Directory.EnumerateFiles(subDir, "*", SearchOption.AllDirectories))
RejectReparsePoint(subDir);

foreach (var file in EnumerateResourceFiles(subDir))
{
var relativePath = Path.GetRelativePath(skillDirectory, file)
.Replace('\\', '/');
resources.Add(new ScannedResource(relativePath, file));
resources.Add(new ScannedResource(relativePath, file, GetUnixMode(file)));
}
}

return resources;
}

private static IEnumerable<string> EnumerateResourceFiles(string directory)
{
foreach (var childDirectory in Directory.EnumerateDirectories(directory))
{
RejectReparsePoint(childDirectory);

foreach (var file in EnumerateResourceFiles(childDirectory))
yield return file;
}

foreach (var file in Directory.EnumerateFiles(directory))
{
RejectReparsePoint(file);
yield return file;
}
}

private static void RejectReparsePoint(string path)
{
if ((File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0)
throw new InvalidOperationException($"Skill resources cannot include symbolic links or reparse points: {path}");
}

private static int? GetUnixMode(string file)
{
if (OperatingSystem.IsWindows())
return null;

return (int)(File.GetUnixFileMode(file) & PermissionMask);
}

internal static Dictionary<string, string>? ParseFrontmatter(string content)
{
var match = FrontmatterRegex().Match(content);
Expand Down
10 changes: 5 additions & 5 deletions src/SkillServer/Data/SkillRepository.cs
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ public async Task<IReadOnlyList<SkillFile>> GetFilesAsync(long versionId, Cancel
var files = await connection.QueryAsync<SkillFile>(
"""
SELECT id AS Id, skill_version_id AS SkillVersionId, relative_path AS RelativePath,
sha256 AS Sha256, size_bytes AS SizeBytes
sha256 AS Sha256, size_bytes AS SizeBytes, unix_mode AS UnixMode
FROM skill_files
WHERE skill_version_id = @versionId
ORDER BY relative_path
Expand All @@ -323,15 +323,15 @@ ORDER BY relative_path
return files.ToList();
}

public async Task AddFileAsync(long versionId, string relativePath, string sha256, long sizeBytes, CancellationToken ct = default)
public async Task AddFileAsync(long versionId, string relativePath, string sha256, long sizeBytes, CancellationToken ct = default, int? unixMode = null)
{
await using var connection = new SqliteConnection(_connectionString);
await connection.ExecuteAsync(
"""
INSERT INTO skill_files (skill_version_id, relative_path, sha256, size_bytes)
VALUES (@versionId, @relativePath, @sha256, @sizeBytes)
INSERT INTO skill_files (skill_version_id, relative_path, sha256, size_bytes, unix_mode)
VALUES (@versionId, @relativePath, @sha256, @sizeBytes, @unixMode)
""",
new { versionId, relativePath, sha256, sizeBytes });
new { versionId, relativePath, sha256, sizeBytes, unixMode });
}

public async Task<IReadOnlyList<SkillVersionWithMetadata>> SearchSkillsAsync(
Expand Down
94 changes: 90 additions & 4 deletions src/SkillServer/Endpoints.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
// </copyright>
// -----------------------------------------------------------------------
using Microsoft.AspNetCore.Mvc;
using System.Text.Json;
using SkillServer.Data;
using SkillServer.Models;
using SkillServer.Services;
Expand Down Expand Up @@ -362,8 +363,12 @@ private static async Task<IResult> UploadSkill(
});
}

if (!TryParseResourceMetadata(request.Form, out var resourceMetadata, out var metadataError))
return Results.BadRequest(metadataError);

var resourceFiles = request.Form.Files.GetFiles("resources");
var resources = new List<(ResourcePath Path, Stream Content)>();
var resources = new List<SkillResourceUpload>();
var resourcePaths = new HashSet<string>(StringComparer.Ordinal);
try
{
foreach (var resourceFile in resourceFiles)
Expand All @@ -377,7 +382,20 @@ private static async Task<IResult> UploadSkill(
});
}

resources.Add((resourcePath.Value, resourceFile.OpenReadStream()));
var resourcePathValue = resourcePath.Value.Value;
resourcePaths.Add(resourcePathValue);
resourceMetadata.TryGetValue(resourcePathValue, out var unixMode);
resources.Add(new SkillResourceUpload(resourcePath.Value, resourceFile.OpenReadStream(), unixMode));
}

var unmatchedMetadataPath = resourceMetadata.Keys.FirstOrDefault(path => !resourcePaths.Contains(path));
if (unmatchedMetadataPath is not null)
{
return Results.BadRequest(new ErrorResponse
{
Error = "invalid_resource_metadata",
Message = $"Resource metadata path '{unmatchedMetadataPath}' does not match any uploaded resource."
});
}

await using var stream = file.OpenReadStream();
Expand All @@ -388,11 +406,79 @@ private static async Task<IResult> UploadSkill(
}
finally
{
foreach (var (_, content) in resources)
await content.DisposeAsync();
foreach (var resource in resources)
await resource.Content.DisposeAsync();
}
}

private static bool TryParseResourceMetadata(
IFormCollection form,
out Dictionary<string, int?> metadata,
out ErrorResponse? error)
{
metadata = new Dictionary<string, int?>(StringComparer.Ordinal);
error = null;

var raw = form["resourceMetadata"].FirstOrDefault();
if (string.IsNullOrWhiteSpace(raw))
return true;

IReadOnlyList<SkillResourceUploadMetadata>? entries;
try
{
entries = JsonSerializer.Deserialize(
raw,
SkillServerJsonContext.Default.IReadOnlyListSkillResourceUploadMetadata);
}
catch (JsonException ex)
{
error = new ErrorResponse
{
Error = "invalid_resource_metadata",
Message = $"Invalid resourceMetadata JSON: {ex.Message}"
};
return false;
}

if (entries is null)
return true;

foreach (var entry in entries)
{
if (!ResourcePath.TryCreate(entry.Path, out var resourcePath))
{
error = new ErrorResponse
{
Error = "invalid_resource_metadata",
Message = $"Invalid resource metadata path: '{entry.Path}'."
};
return false;
}

if (entry.UnixMode is { } unixMode && !SkillArchiveBuilder.IsSafeUnixMode(unixMode))
{
error = new ErrorResponse
{
Error = "invalid_resource_metadata",
Message = $"Invalid unixMode for resource '{entry.Path}'. Must contain only standard permission bits between 0 and 511."
};
return false;
}

if (!metadata.TryAdd(resourcePath.Value.Value, entry.UnixMode))
{
error = new ErrorResponse
{
Error = "invalid_resource_metadata",
Message = $"Duplicate resource metadata path: '{entry.Path}'."
};
return false;
}
}

return true;
}

private static IResult HandleUploadResult(SkillUploadResult result, IConfiguration configuration)
{
if (!result.Success)
Expand Down
Loading
Loading