diff --git a/src/Netclaw.SkillClient/Models.cs b/src/Netclaw.SkillClient/Models.cs index 6178d88..d64b42a 100644 --- a/src/Netclaw.SkillClient/Models.cs +++ b/src/Netclaw.SkillClient/Models.cs @@ -59,6 +59,20 @@ public sealed record RfcResourceEntry [JsonPropertyName("url")] public string Url { get; init; } = ""; + + [JsonPropertyName("unixMode")] + public int? UnixMode { get; init; } +} + +public sealed record SkillResourceUpload(string RelativePath, Stream Content, int? UnixMode = null); + +public sealed record SkillResourceUploadMetadata +{ + [JsonPropertyName("path")] + public string Path { get; init; } = ""; + + [JsonPropertyName("unixMode")] + public int? UnixMode { get; init; } } /// @@ -244,6 +258,7 @@ public sealed record ErrorResponse [JsonSerializable(typeof(IReadOnlyList))] [JsonSerializable(typeof(IReadOnlyList))] [JsonSerializable(typeof(IReadOnlyList))] +[JsonSerializable(typeof(IReadOnlyList))] [JsonSerializable(typeof(CreateApiKeyRequest))] [JsonSerializable(typeof(ErrorResponse))] [JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] diff --git a/src/Netclaw.SkillClient/SkillServerClient.cs b/src/Netclaw.SkillClient/SkillServerClient.cs index 7558a17..e5f7f1d 100644 --- a/src/Netclaw.SkillClient/SkillServerClient.cs +++ b/src/Netclaw.SkillClient/SkillServerClient.cs @@ -194,13 +194,27 @@ public async Task UploadSkillAsync( string name, string version, Stream skillMdContent, string? category = null, CancellationToken ct = default) { - return await UploadSkillWithResourcesAsync(name, version, skillMdContent, [], category, ct); + return await UploadSkillWithResourceUploadsAsync(name, version, skillMdContent, Array.Empty(), category, ct); } public async Task UploadSkillWithResourcesAsync( string name, string version, Stream skillMdContent, IReadOnlyList<(string RelativePath, Stream Content)> resources, string? category = null, CancellationToken ct = default) + { + return await UploadSkillWithResourceUploadsAsync( + name, + version, + skillMdContent, + resources.Select(r => new SkillResourceUpload(r.RelativePath, r.Content)).ToList(), + category, + ct); + } + + public async Task UploadSkillWithResourceUploadsAsync( + string name, string version, Stream skillMdContent, + IReadOnlyList resources, + string? category = null, CancellationToken ct = default) { using var response = await PostSkillAsync(name, version, skillMdContent, resources, category, ct); response.EnsureSuccessStatusCode(); @@ -212,6 +226,20 @@ public async Task UploadSkillWithResourcesAsync( string name, string version, Stream skillMdContent, IReadOnlyList<(string RelativePath, Stream Content)> resources, string? category = null, CancellationToken ct = default) + { + return await UploadSkillIfNotExistsWithResourceUploadsAsync( + name, + version, + skillMdContent, + resources.Select(r => new SkillResourceUpload(r.RelativePath, r.Content)).ToList(), + category, + ct); + } + + public async Task UploadSkillIfNotExistsWithResourceUploadsAsync( + string name, string version, Stream skillMdContent, + IReadOnlyList resources, + string? category = null, CancellationToken ct = default) { using var response = await PostSkillAsync(name, version, skillMdContent, resources, category, ct); @@ -225,7 +253,7 @@ public async Task UploadSkillWithResourcesAsync( private async Task PostSkillAsync( string name, string version, Stream skillMdContent, - IReadOnlyList<(string RelativePath, Stream Content)> resources, + IReadOnlyList resources, string? category, CancellationToken ct) { using var content = new MultipartFormDataContent(); @@ -235,8 +263,24 @@ private async Task PostSkillAsync( content.Add(new StringContent(category), "category"); content.Add(new StreamContent(skillMdContent), "file", "SKILL.md"); - foreach (var (relativePath, resourceStream) in resources) - content.Add(new StreamContent(resourceStream), "resources", relativePath); + foreach (var resource in resources) + content.Add(new StreamContent(resource.Content), "resources", resource.RelativePath); + + if (resources.Any(resource => resource.UnixMode.HasValue)) + { + var metadata = resources + .Where(resource => resource.UnixMode.HasValue) + .Select(resource => new SkillResourceUploadMetadata + { + Path = resource.RelativePath, + UnixMode = resource.UnixMode + }) + .ToList(); + var json = JsonSerializer.Serialize( + metadata, + SkillServerClientJsonContext.Default.IReadOnlyListSkillResourceUploadMetadata); + content.Add(new StringContent(json), "resourceMetadata"); + } return await _httpClient.PostAsync("skills", content, ct); } diff --git a/src/Netclaw.SkillServer.Cli/Publishing/PublishOrchestrator.cs b/src/Netclaw.SkillServer.Cli/Publishing/PublishOrchestrator.cs index eaf2003..801a109 100644 --- a/src/Netclaw.SkillServer.Cli/Publishing/PublishOrchestrator.cs +++ b/src/Netclaw.SkillServer.Cli/Publishing/PublishOrchestrator.cs @@ -67,7 +67,7 @@ public async Task PublishAsync( } } - var resources = new List<(string RelativePath, Stream Content)>(); + var resources = new List(); FileStream? skillMdStream = null; try { @@ -75,17 +75,20 @@ public async Task PublishAsync( foreach (var resource in skill.Resources) { - resources.Add((resource.RelativePath, File.OpenRead(resource.AbsolutePath))); + resources.Add(new SkillResourceUpload( + resource.RelativePath, + File.OpenRead(resource.AbsolutePath), + resource.UnixMode)); } if (options.Verbose) { ConsoleOutput.WriteDim($" Uploading SKILL.md ({FormatSize(skillMdStream.Length)})"); - foreach (var (relativePath, stream) in resources) - ConsoleOutput.WriteDim($" Uploading {relativePath} ({FormatSize(stream.Length)})"); + foreach (var resource in resources) + ConsoleOutput.WriteDim($" Uploading {resource.RelativePath} ({FormatSize(resource.Content.Length)})"); } - var uploadResponse = await _client.UploadSkillIfNotExistsAsync( + var uploadResponse = await _client.UploadSkillIfNotExistsWithResourceUploadsAsync( skill.Name, version, skillMdStream, resources, skill.Category, ct); if (uploadResponse is null) @@ -106,8 +109,8 @@ public async Task PublishAsync( { if (skillMdStream is not null) await skillMdStream.DisposeAsync(); - foreach (var (_, stream) in resources) - await stream.DisposeAsync(); + foreach (var resource in resources) + await resource.Content.DisposeAsync(); } } diff --git a/src/Netclaw.SkillServer.Cli/Publishing/SkillDirectoryScanner.cs b/src/Netclaw.SkillServer.Cli/Publishing/SkillDirectoryScanner.cs index bb3a40d..0d5169e 100644 --- a/src/Netclaw.SkillServer.Cli/Publishing/SkillDirectoryScanner.cs +++ b/src/Netclaw.SkillServer.Cli/Publishing/SkillDirectoryScanner.cs @@ -17,10 +17,20 @@ internal sealed record ScannedSkill( string SkillMdPath, IReadOnlyList Resources); -internal readonly record struct ScannedResource(string RelativePath, string AbsolutePath); +internal readonly record struct ScannedResource(string RelativePath, string AbsolutePath, int? UnixMode); internal static partial class SkillDirectoryScanner { + private const UnixFileMode PermissionMask = UnixFileMode.UserRead + | UnixFileMode.UserWrite + | UnixFileMode.UserExecute + | UnixFileMode.GroupRead + | UnixFileMode.GroupWrite + | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead + | UnixFileMode.OtherWrite + | UnixFileMode.OtherExecute; + [GeneratedRegex(@"^---\s*\n(.*?)\n---", RegexOptions.Singleline)] private static partial Regex FrontmatterRegex(); @@ -75,17 +85,50 @@ private static IReadOnlyList ScanResources(string skillDirector foreach (var subDir in Directory.EnumerateDirectories(skillDirectory)) { - foreach (var file in Directory.EnumerateFiles(subDir, "*", SearchOption.AllDirectories)) + RejectReparsePoint(subDir); + + foreach (var file in EnumerateResourceFiles(subDir)) { var relativePath = Path.GetRelativePath(skillDirectory, file) .Replace('\\', '/'); - resources.Add(new ScannedResource(relativePath, file)); + resources.Add(new ScannedResource(relativePath, file, GetUnixMode(file))); } } return resources; } + private static IEnumerable EnumerateResourceFiles(string directory) + { + foreach (var childDirectory in Directory.EnumerateDirectories(directory)) + { + RejectReparsePoint(childDirectory); + + foreach (var file in EnumerateResourceFiles(childDirectory)) + yield return file; + } + + foreach (var file in Directory.EnumerateFiles(directory)) + { + RejectReparsePoint(file); + yield return file; + } + } + + private static void RejectReparsePoint(string path) + { + if ((File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0) + throw new InvalidOperationException($"Skill resources cannot include symbolic links or reparse points: {path}"); + } + + private static int? GetUnixMode(string file) + { + if (OperatingSystem.IsWindows()) + return null; + + return (int)(File.GetUnixFileMode(file) & PermissionMask); + } + internal static Dictionary? ParseFrontmatter(string content) { var match = FrontmatterRegex().Match(content); diff --git a/src/SkillServer/Data/SkillRepository.cs b/src/SkillServer/Data/SkillRepository.cs index 7de1f81..f590c22 100644 --- a/src/SkillServer/Data/SkillRepository.cs +++ b/src/SkillServer/Data/SkillRepository.cs @@ -314,7 +314,7 @@ public async Task> GetFilesAsync(long versionId, Cancel var files = await connection.QueryAsync( """ SELECT id AS Id, skill_version_id AS SkillVersionId, relative_path AS RelativePath, - sha256 AS Sha256, size_bytes AS SizeBytes + sha256 AS Sha256, size_bytes AS SizeBytes, unix_mode AS UnixMode FROM skill_files WHERE skill_version_id = @versionId ORDER BY relative_path @@ -323,15 +323,15 @@ ORDER BY relative_path return files.ToList(); } - public async Task AddFileAsync(long versionId, string relativePath, string sha256, long sizeBytes, CancellationToken ct = default) + public async Task AddFileAsync(long versionId, string relativePath, string sha256, long sizeBytes, CancellationToken ct = default, int? unixMode = null) { await using var connection = new SqliteConnection(_connectionString); await connection.ExecuteAsync( """ - INSERT INTO skill_files (skill_version_id, relative_path, sha256, size_bytes) - VALUES (@versionId, @relativePath, @sha256, @sizeBytes) + INSERT INTO skill_files (skill_version_id, relative_path, sha256, size_bytes, unix_mode) + VALUES (@versionId, @relativePath, @sha256, @sizeBytes, @unixMode) """, - new { versionId, relativePath, sha256, sizeBytes }); + new { versionId, relativePath, sha256, sizeBytes, unixMode }); } public async Task> SearchSkillsAsync( diff --git a/src/SkillServer/Endpoints.cs b/src/SkillServer/Endpoints.cs index bbc1c88..4826fd2 100644 --- a/src/SkillServer/Endpoints.cs +++ b/src/SkillServer/Endpoints.cs @@ -4,6 +4,7 @@ // // ----------------------------------------------------------------------- using Microsoft.AspNetCore.Mvc; +using System.Text.Json; using SkillServer.Data; using SkillServer.Models; using SkillServer.Services; @@ -362,8 +363,12 @@ private static async Task UploadSkill( }); } + if (!TryParseResourceMetadata(request.Form, out var resourceMetadata, out var metadataError)) + return Results.BadRequest(metadataError); + var resourceFiles = request.Form.Files.GetFiles("resources"); - var resources = new List<(ResourcePath Path, Stream Content)>(); + var resources = new List(); + var resourcePaths = new HashSet(StringComparer.Ordinal); try { foreach (var resourceFile in resourceFiles) @@ -377,7 +382,20 @@ private static async Task UploadSkill( }); } - resources.Add((resourcePath.Value, resourceFile.OpenReadStream())); + var resourcePathValue = resourcePath.Value.Value; + resourcePaths.Add(resourcePathValue); + resourceMetadata.TryGetValue(resourcePathValue, out var unixMode); + resources.Add(new SkillResourceUpload(resourcePath.Value, resourceFile.OpenReadStream(), unixMode)); + } + + var unmatchedMetadataPath = resourceMetadata.Keys.FirstOrDefault(path => !resourcePaths.Contains(path)); + if (unmatchedMetadataPath is not null) + { + return Results.BadRequest(new ErrorResponse + { + Error = "invalid_resource_metadata", + Message = $"Resource metadata path '{unmatchedMetadataPath}' does not match any uploaded resource." + }); } await using var stream = file.OpenReadStream(); @@ -388,11 +406,79 @@ private static async Task UploadSkill( } finally { - foreach (var (_, content) in resources) - await content.DisposeAsync(); + foreach (var resource in resources) + await resource.Content.DisposeAsync(); } } + private static bool TryParseResourceMetadata( + IFormCollection form, + out Dictionary metadata, + out ErrorResponse? error) + { + metadata = new Dictionary(StringComparer.Ordinal); + error = null; + + var raw = form["resourceMetadata"].FirstOrDefault(); + if (string.IsNullOrWhiteSpace(raw)) + return true; + + IReadOnlyList? entries; + try + { + entries = JsonSerializer.Deserialize( + raw, + SkillServerJsonContext.Default.IReadOnlyListSkillResourceUploadMetadata); + } + catch (JsonException ex) + { + error = new ErrorResponse + { + Error = "invalid_resource_metadata", + Message = $"Invalid resourceMetadata JSON: {ex.Message}" + }; + return false; + } + + if (entries is null) + return true; + + foreach (var entry in entries) + { + if (!ResourcePath.TryCreate(entry.Path, out var resourcePath)) + { + error = new ErrorResponse + { + Error = "invalid_resource_metadata", + Message = $"Invalid resource metadata path: '{entry.Path}'." + }; + return false; + } + + if (entry.UnixMode is { } unixMode && !SkillArchiveBuilder.IsSafeUnixMode(unixMode)) + { + error = new ErrorResponse + { + Error = "invalid_resource_metadata", + Message = $"Invalid unixMode for resource '{entry.Path}'. Must contain only standard permission bits between 0 and 511." + }; + return false; + } + + if (!metadata.TryAdd(resourcePath.Value.Value, entry.UnixMode)) + { + error = new ErrorResponse + { + Error = "invalid_resource_metadata", + Message = $"Duplicate resource metadata path: '{entry.Path}'." + }; + return false; + } + } + + return true; + } + private static IResult HandleUploadResult(SkillUploadResult result, IConfiguration configuration) { if (!result.Success) diff --git a/src/SkillServer/Models/ApiModels.cs b/src/SkillServer/Models/ApiModels.cs index f0eab30..a12648c 100644 --- a/src/SkillServer/Models/ApiModels.cs +++ b/src/SkillServer/Models/ApiModels.cs @@ -43,6 +43,15 @@ public sealed record SkillUploadResponse public required string Url { get; init; } } +public sealed record SkillResourceUploadMetadata +{ + [JsonPropertyName("path")] + public required string Path { get; init; } + + [JsonPropertyName("unixMode")] + public int? UnixMode { get; init; } +} + public sealed record SubAgentUploadResponse { [JsonPropertyName("name")] diff --git a/src/SkillServer/Models/RfcIndex.cs b/src/SkillServer/Models/RfcIndex.cs index 559a6aa..9753950 100644 --- a/src/SkillServer/Models/RfcIndex.cs +++ b/src/SkillServer/Models/RfcIndex.cs @@ -61,4 +61,8 @@ public sealed record RfcResourceEntry [JsonPropertyName("url")] public required string Url { get; init; } + + [JsonPropertyName("unixMode")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? UnixMode { get; init; } } diff --git a/src/SkillServer/Models/Skill.cs b/src/SkillServer/Models/Skill.cs index f49b806..d2bd773 100644 --- a/src/SkillServer/Models/Skill.cs +++ b/src/SkillServer/Models/Skill.cs @@ -46,6 +46,7 @@ public sealed record SkillFile public required string RelativePath { get; init; } public required string Sha256 { get; init; } public required long SizeBytes { get; init; } + public int? UnixMode { get; init; } } /// diff --git a/src/SkillServer/Models/SkillServerJsonContext.cs b/src/SkillServer/Models/SkillServerJsonContext.cs index 4d01e7b..fa0b471 100644 --- a/src/SkillServer/Models/SkillServerJsonContext.cs +++ b/src/SkillServer/Models/SkillServerJsonContext.cs @@ -15,6 +15,7 @@ namespace SkillServer.Models; [JsonSerializable(typeof(RfcResourceEntry))] [JsonSerializable(typeof(SkillUploadRequest))] [JsonSerializable(typeof(SkillUploadResponse))] +[JsonSerializable(typeof(IReadOnlyList))] [JsonSerializable(typeof(SubAgentUploadResponse))] [JsonSerializable(typeof(SkillSummary))] [JsonSerializable(typeof(SkillVersionSummary))] diff --git a/src/SkillServer/Models/ValueObjects.cs b/src/SkillServer/Models/ValueObjects.cs index 9f545f0..dd0feb1 100644 --- a/src/SkillServer/Models/ValueObjects.cs +++ b/src/SkillServer/Models/ValueObjects.cs @@ -155,13 +155,18 @@ public static bool TryCreate(string? value, [NotNullWhen(true)] out ResourcePath if (string.IsNullOrWhiteSpace(value)) return false; - if (value.Contains("..") || value.StartsWith('/') || value.StartsWith('\\')) + var normalized = value.Replace('\\', '/'); + + if (normalized.StartsWith('/') || normalized.Contains(':') || normalized.Contains('\0')) return false; - var normalized = value.Replace('\\', '/'); + var segments = normalized.Split('/'); + + // Must be in a subdirectory — bare filenames at the root are not resources. + if (segments.Length < 2) + return false; - // Must be in a subdirectory — bare filenames at the root are not resources - if (!normalized.Contains('/') || normalized.IndexOf('/') == normalized.Length - 1) + if (segments.Any(segment => segment.Length == 0 || segment is "." or "..")) return false; result = new ResourcePath(normalized); diff --git a/src/SkillServer/Services/IndexGenerator.cs b/src/SkillServer/Services/IndexGenerator.cs index 8b008b2..5d0ad90 100644 --- a/src/SkillServer/Services/IndexGenerator.cs +++ b/src/SkillServer/Services/IndexGenerator.cs @@ -56,7 +56,8 @@ public async Task GenerateRfcIndexAsync(CancellationToken ct = de { Path = f.RelativePath, Digest = Sha256Digest.Create(f.Sha256).Value, - Url = $"{baseUrl}/skills/{v.SkillName}/{v.Version}/{f.RelativePath}" + Url = $"{baseUrl}/skills/{v.SkillName}/{v.Version}/{f.RelativePath}", + UnixMode = f.UnixMode }).ToList() : null }); diff --git a/src/SkillServer/Services/SkillArchiveBackfillService.cs b/src/SkillServer/Services/SkillArchiveBackfillService.cs index 33c8c8d..ee074e9 100644 --- a/src/SkillServer/Services/SkillArchiveBackfillService.cs +++ b/src/SkillServer/Services/SkillArchiveBackfillService.cs @@ -48,7 +48,7 @@ private async Task BackfillVersionAsync(SkillVersion version, CancellationToken } var files = await _repository.GetFilesAsync(version.Id, ct); - var resources = new List<(ResourcePath Path, byte[] Content)>(files.Count); + var resources = new List(files.Count); foreach (var file in files) { @@ -65,7 +65,7 @@ private async Task BackfillVersionAsync(SkillVersion version, CancellationToken return; } - resources.Add((resourcePath.Value, content)); + resources.Add(new SkillArchiveResource(resourcePath.Value, content, file.UnixMode)); } var archiveBytes = SkillArchiveBuilder.BuildZip(skillMdBytes, resources); diff --git a/src/SkillServer/Services/SkillArchiveBuilder.cs b/src/SkillServer/Services/SkillArchiveBuilder.cs index c19a001..e807d6c 100644 --- a/src/SkillServer/Services/SkillArchiveBuilder.cs +++ b/src/SkillServer/Services/SkillArchiveBuilder.cs @@ -10,33 +10,50 @@ namespace SkillServer.Services; internal static class SkillArchiveBuilder { + private const int RegularFileType = 0x8000; + private const int DefaultFileMode = 0x1A4; // 0644 + internal const int PermissionBitsMask = 0x1FF; private static readonly DateTimeOffset FixedTimestamp = new(1980, 1, 1, 0, 0, 0, TimeSpan.Zero); public static byte[] BuildZip( byte[] skillMdContent, - IReadOnlyList<(ResourcePath Path, byte[] Content)> resources) + IReadOnlyList resources) { using var archiveStream = new MemoryStream(); using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create, leaveOpen: true)) { - AddEntry(archive, "SKILL.md", skillMdContent); + AddEntry(archive, "SKILL.md", skillMdContent, unixMode: null); foreach (var resource in resources.OrderBy(r => r.Path.Value, StringComparer.Ordinal)) { - AddEntry(archive, resource.Path.Value, resource.Content); + AddEntry(archive, resource.Path.Value, resource.Content, resource.UnixMode); } } return archiveStream.ToArray(); } - private static void AddEntry(ZipArchive archive, string path, byte[] content) + private static void AddEntry(ZipArchive archive, string path, byte[] content, int? unixMode) { var entry = archive.CreateEntry(path, CompressionLevel.NoCompression); entry.LastWriteTime = FixedTimestamp; - entry.ExternalAttributes = 0; + entry.ExternalAttributes = ToZipExternalAttributes(unixMode); using var entryStream = entry.Open(); entryStream.Write(content); } + + internal static int ToZipExternalAttributes(int? unixMode) + { + var mode = NormalizeUnixMode(unixMode); + return unchecked((int)((RegularFileType | mode) << 16)); + } + + internal static bool IsSafeUnixMode(int unixMode) + => unixMode >= 0 && (unixMode & ~PermissionBitsMask) == 0; + + internal static int NormalizeUnixMode(int? unixMode) + => (unixMode ?? DefaultFileMode) & PermissionBitsMask; } + +internal readonly record struct SkillArchiveResource(ResourcePath Path, byte[] Content, int? UnixMode); diff --git a/src/SkillServer/Services/SkillUploadService.cs b/src/SkillServer/Services/SkillUploadService.cs index 9527a5c..e07ebf9 100644 --- a/src/SkillServer/Services/SkillUploadService.cs +++ b/src/SkillServer/Services/SkillUploadService.cs @@ -131,7 +131,7 @@ public async Task UploadSkillWithResourcesAsync( SkillName name, SkillVersionString version, Stream skillMdContent, - IReadOnlyList<(ResourcePath Path, Stream Content)> resources, + IReadOnlyList resources, string? category = null, CancellationToken ct = default) { @@ -142,12 +142,12 @@ public async Task UploadSkillWithResourcesAsync( await skillMdContent.CopyToAsync(skillMdBuffer, ct); var skillMdBytes = skillMdBuffer.ToArray(); - var resourceContents = new List<(ResourcePath Path, byte[] Content)>(resources.Count); - foreach (var (resourcePath, content) in resources) + var resourceContents = new List(resources.Count); + foreach (var resource in resources) { using var resourceBuffer = new MemoryStream(); - await content.CopyToAsync(resourceBuffer, ct); - resourceContents.Add((resourcePath, resourceBuffer.ToArray())); + await resource.Content.CopyToAsync(resourceBuffer, ct); + resourceContents.Add(new SkillArchiveResource(resource.Path, resourceBuffer.ToArray(), resource.UnixMode)); } // First upload the SKILL.md @@ -164,12 +164,12 @@ public async Task UploadSkillWithResourcesAsync( if (skillVersion is null) return SkillUploadResult.Failed("Version not found after upload."); // Store resource files - foreach (var (resourcePath, contentBytes) in resourceContents) + foreach (var resource in resourceContents) { - var (digest, sizeBytes) = await _blobStorage.StoreAsync(contentBytes, ct); + var (digest, sizeBytes) = await _blobStorage.StoreAsync(resource.Content, ct); var parsedDigest = Sha256Digest.Create(digest); - await _repository.AddFileAsync(skillVersion.Id, resourcePath.Value, parsedDigest.Value, sizeBytes, ct); - _logger.LogDebug("Added resource {Path} ({Digest})", resourcePath.Value, parsedDigest.Value); + await _repository.AddFileAsync(skillVersion.Id, resource.Path.Value, parsedDigest.Value, sizeBytes, ct, resource.UnixMode); + _logger.LogDebug("Added resource {Path} ({Digest})", resource.Path.Value, parsedDigest.Value); } var archiveBytes = SkillArchiveBuilder.BuildZip(skillMdBytes, resourceContents); @@ -205,6 +205,8 @@ await _repository.UpdateVersionArtifactAsync( private static partial Regex FrontmatterRegex(); } +public readonly record struct SkillResourceUpload(ResourcePath Path, Stream Content, int? UnixMode); + /// /// Result of a skill upload operation. /// diff --git a/src/SkillServer/migrations/006_skill_file_unix_mode.sql b/src/SkillServer/migrations/006_skill_file_unix_mode.sql new file mode 100644 index 0000000..c528fab --- /dev/null +++ b/src/SkillServer/migrations/006_skill_file_unix_mode.sql @@ -0,0 +1 @@ +ALTER TABLE skill_files ADD COLUMN unix_mode INTEGER NULL; diff --git a/tests/Netclaw.SkillServer.Cli.Tests/SkillDirectoryScannerTests.cs b/tests/Netclaw.SkillServer.Cli.Tests/SkillDirectoryScannerTests.cs index 78f41e3..878f17b 100644 --- a/tests/Netclaw.SkillServer.Cli.Tests/SkillDirectoryScannerTests.cs +++ b/tests/Netclaw.SkillServer.Cli.Tests/SkillDirectoryScannerTests.cs @@ -78,6 +78,128 @@ public void ScanDirectory_WithResources_FindsResourceFiles() Assert.Contains(result.Resources, r => r.RelativePath == "references/config.md"); } + [Fact] + public void ScanDirectory_CapturesUnixResourceMode() + { + if (OperatingSystem.IsWindows()) + return; + + var skillDir = Path.Combine(_tempDir, "skill-with-executable"); + Directory.CreateDirectory(skillDir); + File.WriteAllText(Path.Combine(skillDir, "SKILL.md"), """ + --- + name: skill-with-executable + version: 1.0.0 + description: Has executable resource + --- + # Skill + """); + + var binDir = Path.Combine(skillDir, "bin"); + Directory.CreateDirectory(binDir); + var toolPath = Path.Combine(binDir, "tool"); + File.WriteAllText(toolPath, "#!/bin/sh\necho ok\n"); + File.SetUnixFileMode(toolPath, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute); + + var result = SkillDirectoryScanner.ScanDirectory(skillDir); + + Assert.NotNull(result); + var resource = Assert.Single(result.Resources); + Assert.Equal("bin/tool", resource.RelativePath); + Assert.Equal(0x1ED, resource.UnixMode); + } + + [Fact] + public void ScanDirectory_StripsSpecialUnixModeBits() + { + if (OperatingSystem.IsWindows()) + return; + + var skillDir = Path.Combine(_tempDir, "skill-with-special-mode"); + Directory.CreateDirectory(skillDir); + File.WriteAllText(Path.Combine(skillDir, "SKILL.md"), """ + --- + name: skill-with-special-mode + version: 1.0.0 + description: Has special mode bits + --- + # Skill + """); + + var binDir = Path.Combine(skillDir, "bin"); + Directory.CreateDirectory(binDir); + var toolPath = Path.Combine(binDir, "tool"); + File.WriteAllText(toolPath, "#!/bin/sh\necho ok\n"); + File.SetUnixFileMode(toolPath, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute | + UnixFileMode.SetUser | UnixFileMode.SetGroup | UnixFileMode.StickyBit); + + var result = SkillDirectoryScanner.ScanDirectory(skillDir); + + Assert.NotNull(result); + var resource = Assert.Single(result.Resources); + Assert.Equal("bin/tool", resource.RelativePath); + Assert.Equal(0x1ED, resource.UnixMode); + } + + [Fact] + public void ScanDirectory_RejectsSymlinkResourceFiles() + { + if (OperatingSystem.IsWindows()) + return; + + var skillDir = Path.Combine(_tempDir, "skill-with-symlink-file"); + Directory.CreateDirectory(skillDir); + File.WriteAllText(Path.Combine(skillDir, "SKILL.md"), """ + --- + name: skill-with-symlink-file + version: 1.0.0 + description: Has symlink resource + --- + # Skill + """); + + var refDir = Path.Combine(skillDir, "references"); + Directory.CreateDirectory(refDir); + var targetPath = Path.Combine(_tempDir, "outside.txt"); + File.WriteAllText(targetPath, "outside"); + File.CreateSymbolicLink(Path.Combine(refDir, "outside.txt"), targetPath); + + var ex = Assert.Throws(() => SkillDirectoryScanner.ScanDirectory(skillDir)); + Assert.Contains("symbolic links or reparse points", ex.Message); + } + + [Fact] + public void ScanDirectory_RejectsSymlinkResourceDirectories() + { + if (OperatingSystem.IsWindows()) + return; + + var skillDir = Path.Combine(_tempDir, "skill-with-symlink-dir"); + Directory.CreateDirectory(skillDir); + File.WriteAllText(Path.Combine(skillDir, "SKILL.md"), """ + --- + name: skill-with-symlink-dir + version: 1.0.0 + description: Has symlink directory + --- + # Skill + """); + + var outsideDir = Path.Combine(_tempDir, "outside-dir"); + Directory.CreateDirectory(outsideDir); + File.WriteAllText(Path.Combine(outsideDir, "outside.txt"), "outside"); + Directory.CreateSymbolicLink(Path.Combine(skillDir, "references"), outsideDir); + + var ex = Assert.Throws(() => SkillDirectoryScanner.ScanDirectory(skillDir)); + Assert.Contains("symbolic links or reparse points", ex.Message); + } + [Fact] public void ScanDirectory_MissingSkillMd_ReturnsNull() { diff --git a/tests/SkillServer.Integration.Tests/SkillServerIntegrationTests.cs b/tests/SkillServer.Integration.Tests/SkillServerIntegrationTests.cs index 6882f9d..12c1325 100644 --- a/tests/SkillServer.Integration.Tests/SkillServerIntegrationTests.cs +++ b/tests/SkillServer.Integration.Tests/SkillServerIntegrationTests.cs @@ -917,6 +917,7 @@ public async Task UploadSkillWithResources_EndToEnd() # Resource Upload Test See references/guide.md for details. + Run tools/check for local verification. """; var referenceContent = """ @@ -936,6 +937,11 @@ Second section content. echo "setup complete" """; + var toolContent = """ + #!/bin/bash + echo "tool check complete" + """; + using var content = new MultipartFormDataContent(); content.Add(new StringContent(skillName), "name"); content.Add(new StringContent("1.0.0"), "version"); @@ -952,12 +958,22 @@ Second section content. scriptFileContent.Headers.ContentType = new MediaTypeHeaderValue("application/x-sh"); content.Add(scriptFileContent, "resources", "scripts/setup.sh"); + var toolFileContent = new ByteArrayContent(Encoding.UTF8.GetBytes(toolContent)); + toolFileContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + content.Add(toolFileContent, "resources", "tools/check"); + + var resourceMetadata = JsonSerializer.Serialize(new[] + { + new { path = "tools/check", unixMode = 0x1ED } + }); + content.Add(new StringContent(resourceMetadata, Encoding.UTF8, "application/json"), "resourceMetadata"); + var uploadResponse = await _fixture.AuthenticatedHttpClient.PostAsync("/skills", content, ct); Assert.Equal(HttpStatusCode.Created, uploadResponse.StatusCode); var version = await _fixture.Client.GetVersionAsync(skillName, "1.0.0", ct); Assert.NotNull(version); - Assert.Equal(2, version.FileCount); + Assert.Equal(3, version.FileCount); var downloadedSkill = await _fixture.Client.GetSkillFileAsStringAsync(skillName, "1.0.0", ct: ct); Assert.Contains("# Resource Upload Test", downloadedSkill); @@ -975,6 +991,12 @@ Second section content. var scriptBody = await scriptResponse.Content.ReadAsStringAsync(ct); Assert.Contains("setup complete", scriptBody); + var toolResponse = await _fixture.HttpClient.GetAsync( + $"/skills/{skillName}/1.0.0/tools/check", ct); + Assert.Equal(HttpStatusCode.OK, toolResponse.StatusCode); + var toolBody = await toolResponse.Content.ReadAsStringAsync(ct); + Assert.Contains("tool check complete", toolBody); + var index = await _fixture.Client.GetRfcIndexAsync(ct); Assert.NotNull(index); var indexSkill = index.Skills.FirstOrDefault(s => s.Name == skillName); @@ -1005,18 +1027,22 @@ Second section content. Assert.Equal([ "SKILL.md", "references/guide.md", - "scripts/setup.sh" + "scripts/setup.sh", + "tools/check" ], archive.Entries.Select(e => e.FullName).ToArray()); + Assert.Equal(0x1ED, GetUnixMode(archive.GetEntry("tools/check")!)); + using var archivedSkillReader = new StreamReader(archive.GetEntry("SKILL.md")!.Open()); var archivedSkill = await archivedSkillReader.ReadToEndAsync(ct); Assert.Contains("# Resource Upload Test", archivedSkill); var resources = indexSkill!.Resources; Assert.NotNull(resources); - Assert.Equal(2, resources!.Count); + Assert.Equal(3, resources!.Count); Assert.Contains(resources, r => r.Path == "references/guide.md"); Assert.Contains(resources, r => r.Path == "scripts/setup.sh"); + Assert.Contains(resources, r => r.Path == "tools/check" && r.UnixMode == 0x1ED); } [Fact] @@ -1080,4 +1106,81 @@ public async Task UploadSkillWithResources_InvalidPath_ReturnsBadRequest() var uploadResponse = await _fixture.AuthenticatedHttpClient.PostAsync("/skills", content, ct); Assert.Equal(HttpStatusCode.BadRequest, uploadResponse.StatusCode); } + + [Theory] + [InlineData("C:/temp/tool")] + [InlineData("C:\\temp\\tool")] + public async Task UploadSkillWithResources_WindowsDrivePath_ReturnsBadRequest(string path) + { + var ct = TestContext.Current.CancellationToken; + var skillName = $"drivepath-{Guid.NewGuid():N}"[..20]; + + using var content = CreateResourceUploadContent(skillName, path, "exploit"); + + var uploadResponse = await _fixture.AuthenticatedHttpClient.PostAsync("/skills", content, ct); + Assert.Equal(HttpStatusCode.BadRequest, uploadResponse.StatusCode); + } + + [Fact] + public async Task UploadSkillWithResources_DangerousUnixMode_ReturnsBadRequest() + { + var ct = TestContext.Current.CancellationToken; + var skillName = $"badmode-{Guid.NewGuid():N}"[..20]; + + using var content = CreateResourceUploadContent(skillName, "tools/check", "#!/bin/sh\necho ok\n"); + var resourceMetadata = JsonSerializer.Serialize(new[] + { + new { path = "tools/check", unixMode = 0xFED } + }); + content.Add(new StringContent(resourceMetadata, Encoding.UTF8, "application/json"), "resourceMetadata"); + + var uploadResponse = await _fixture.AuthenticatedHttpClient.PostAsync("/skills", content, ct); + Assert.Equal(HttpStatusCode.BadRequest, uploadResponse.StatusCode); + } + + [Fact] + public async Task UploadSkillWithResources_UnmatchedResourceMetadata_ReturnsBadRequest() + { + var ct = TestContext.Current.CancellationToken; + var skillName = $"badmetadata-{Guid.NewGuid():N}"[..20]; + + using var content = CreateResourceUploadContent(skillName, "tools/check", "#!/bin/sh\necho ok\n"); + var resourceMetadata = JsonSerializer.Serialize(new[] + { + new { path = "tools/missing", unixMode = 0x1ED } + }); + content.Add(new StringContent(resourceMetadata, Encoding.UTF8, "application/json"), "resourceMetadata"); + + var uploadResponse = await _fixture.AuthenticatedHttpClient.PostAsync("/skills", content, ct); + Assert.Equal(HttpStatusCode.BadRequest, uploadResponse.StatusCode); + } + + private static int GetUnixMode(ZipArchiveEntry entry) + => (entry.ExternalAttributes >> 16) & 0x1FF; + + private static MultipartFormDataContent CreateResourceUploadContent(string skillName, string resourcePath, string resourceContent) + { + var skillContent = $""" + --- + name: {skillName} + description: Testing resource validation + --- + + # Resource Validation Test + """; + + var content = new MultipartFormDataContent(); + content.Add(new StringContent(skillName), "name"); + content.Add(new StringContent("1.0.0"), "version"); + + var fileContent = new ByteArrayContent(Encoding.UTF8.GetBytes(skillContent)); + fileContent.Headers.ContentType = new MediaTypeHeaderValue("text/markdown"); + content.Add(fileContent, "file", "SKILL.md"); + + var resourceFile = new ByteArrayContent(Encoding.UTF8.GetBytes(resourceContent)); + resourceFile.Headers.ContentType = new MediaTypeHeaderValue("text/plain"); + content.Add(resourceFile, "resources", resourcePath); + + return content; + } } diff --git a/tests/SkillServer.Tests/SkillArchiveBackfillServiceTests.cs b/tests/SkillServer.Tests/SkillArchiveBackfillServiceTests.cs index 0112d26..12a7edb 100644 --- a/tests/SkillServer.Tests/SkillArchiveBackfillServiceTests.cs +++ b/tests/SkillServer.Tests/SkillArchiveBackfillServiceTests.cs @@ -63,7 +63,7 @@ public async Task BackfillAsync_ConvertsLegacyResourcefulSkillToArchiveArtifact( skillDigest, skillSizeBytes, ct); - await repository.AddFileAsync(versionId, "references/guide.md", resourceDigest, resourceSizeBytes, ct); + await repository.AddFileAsync(versionId, "references/guide.md", resourceDigest, resourceSizeBytes, ct, unixMode: 0x1ED); await backfillService.BackfillAsync(ct); @@ -79,6 +79,7 @@ public async Task BackfillAsync_ConvertsLegacyResourcefulSkillToArchiveArtifact( using var archive = new ZipArchive(archiveBlob!, ZipArchiveMode.Read); Assert.Equal(["SKILL.md", "references/guide.md"], archive.Entries.Select(e => e.FullName).ToArray()); + Assert.Equal(0x1ED, (archive.GetEntry("references/guide.md")!.ExternalAttributes >> 16) & 0xFFF); } public void Dispose() diff --git a/tests/SkillServer.Tests/SkillArchiveBuilderTests.cs b/tests/SkillServer.Tests/SkillArchiveBuilderTests.cs index 5731e46..9a64ebb 100644 --- a/tests/SkillServer.Tests/SkillArchiveBuilderTests.cs +++ b/tests/SkillServer.Tests/SkillArchiveBuilderTests.cs @@ -20,10 +20,10 @@ public void BuildZip_IsDeterministicAndUsesExpectedLayout() var guide = Encoding.UTF8.GetBytes("# Guide"); var script = Encoding.UTF8.GetBytes("#!/bin/bash\necho setup"); - var resources = new List<(ResourcePath Path, byte[] Content)> + var resources = new List { - (ResourcePath.Create("scripts/setup.sh"), script), - (ResourcePath.Create("references/guide.md"), guide) + new(ResourcePath.Create("scripts/setup.sh"), script, 0x1ED), + new(ResourcePath.Create("references/guide.md"), guide, 0x1A4) }; var reversedResources = resources.AsEnumerable().Reverse().ToList(); @@ -44,5 +44,80 @@ public void BuildZip_IsDeterministicAndUsesExpectedLayout() Assert.All(archive.Entries, entry => Assert.Equal(new DateTimeOffset(1980, 1, 1, 0, 0, 0, TimeSpan.Zero), entry.LastWriteTime)); + + Assert.Equal(0x1A4, GetUnixMode(archive.GetEntry("SKILL.md")!)); + Assert.Equal(0x1A4, GetUnixMode(archive.GetEntry("references/guide.md")!)); + Assert.Equal(0x1ED, GetUnixMode(archive.GetEntry("scripts/setup.sh")!)); } + + [Fact] + public void ToZipExternalAttributes_WritesRegularFileMetadataPortably() + { + var attributes = SkillArchiveBuilder.ToZipExternalAttributes(0x1ED); + + var rawMode = GetRawUnixMode(attributes); + Assert.Equal(0x8000, rawMode & 0xF000); + Assert.Equal(0x1ED, rawMode & 0x1FF); + } + + [Fact] + public void BuildZip_StripsSpecialUnixModeBits() + { + var skillMd = Encoding.UTF8.GetBytes("---\nname: test\ndescription: test\n---\n# Test"); + var script = Encoding.UTF8.GetBytes("#!/bin/sh\necho ok\n"); + + var zip = SkillArchiveBuilder.BuildZip(skillMd, + [ + new SkillArchiveResource(ResourcePath.Create("scripts/tool"), script, 0xFED) + ]); + + using var archiveStream = new MemoryStream(zip); + using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read); + var rawMode = GetRawUnixMode(archive.GetEntry("scripts/tool")!.ExternalAttributes); + + Assert.Equal(0x8000, rawMode & 0xF000); + Assert.Equal(0x1ED, rawMode & 0x1FF); + Assert.Equal(0, rawMode & 0x0E00); + } + + [Fact] + public void BuildZip_WhenExtractedOnUnix_RestoresExecutableBit() + { + if (OperatingSystem.IsWindows()) + return; + + var tempDir = Path.Combine(Path.GetTempPath(), "skillserver-archive-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + + try + { + var skillMd = Encoding.UTF8.GetBytes("---\nname: test\ndescription: test\n---\n# Test"); + var script = Encoding.UTF8.GetBytes("#!/bin/sh\necho ok\n"); + var zip = SkillArchiveBuilder.BuildZip(skillMd, + [ + new SkillArchiveResource(ResourcePath.Create("scripts/tool"), script, 0x1ED) + ]); + + var zipPath = Path.Combine(tempDir, "archive.zip"); + var extractDir = Path.Combine(tempDir, "extracted"); + File.WriteAllBytes(zipPath, zip); + ZipFile.ExtractToDirectory(zipPath, extractDir); + + var mode = (int)(File.GetUnixFileMode(Path.Combine(extractDir, "scripts", "tool")) & + (UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute)); + Assert.Equal(0x1ED, mode); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + private static int GetUnixMode(ZipArchiveEntry entry) + => GetRawUnixMode(entry.ExternalAttributes) & 0x1FF; + + private static int GetRawUnixMode(int externalAttributes) + => (externalAttributes >> 16) & 0xFFFF; } diff --git a/tests/SkillServer.Tests/ValueObjectTests.cs b/tests/SkillServer.Tests/ValueObjectTests.cs index 19e4f9c..d8a040e 100644 --- a/tests/SkillServer.Tests/ValueObjectTests.cs +++ b/tests/SkillServer.Tests/ValueObjectTests.cs @@ -117,8 +117,16 @@ public sealed class ResourcePathTests [InlineData("references/deep/nested/file.md", true)] [InlineData("custom/file.txt", true)] [InlineData("examples/demo.py", true)] + [InlineData("references/hidden..file.md", true)] [InlineData("../secret.txt", false)] + [InlineData("references/../secret.txt", false)] + [InlineData("references/./guide.md", false)] + [InlineData("references//guide.md", false)] [InlineData("/etc/passwd", false)] + [InlineData("C:/temp/tool", false)] + [InlineData("C:\\temp\\tool", false)] + [InlineData("C:temp/tool", false)] + [InlineData("references/guide:latest.md", false)] [InlineData("SKILL.md", false)] // Bare filenames are not resources [InlineData("justadirectory/", false)] [InlineData("", false)]