diff --git a/src/Netclaw.Actors.Tests/Channels/SlackAttachmentIngressTests.cs b/src/Netclaw.Actors.Tests/Channels/SlackAttachmentIngressTests.cs index 997e5b395..40eae1b2e 100644 --- a/src/Netclaw.Actors.Tests/Channels/SlackAttachmentIngressTests.cs +++ b/src/Netclaw.Actors.Tests/Channels/SlackAttachmentIngressTests.cs @@ -40,6 +40,9 @@ public sealed class SlackAttachmentIngressVisionTests : TestKit private static readonly byte[] FakeDocxBytes = "PK\u0003\u0004fake docx content"u8.ToArray(); + private static readonly byte[] FakePlainTextBytes = + "meeting notes\n- discuss Q2 roadmap\n- assign OKRs\n"u8.ToArray(); + private readonly RecordingChatClient _chatClient = new(); private readonly RecordingReplyClient _replyClient = new(); private readonly ConfigurableFakeSlackFileHandler _httpHandler = new(); @@ -125,7 +128,7 @@ private IActorRef BuildGateway( BotUserId: new SlackUserId("UBOT"), DefaultChannelId: null, ReplyClient: _replyClient, - ContentScanner: scanner ?? new NullContentScanner(), + ContentScanner: scanner ?? new MagicByteContentScanner(new ContentPolicy()), ThreadHistoryFetcher: EmptyThreadHistoryFetcher.Instance, AudienceProfiles: profiles, ModelCapabilities: Host.Services.GetRequiredService(), @@ -437,6 +440,57 @@ await AwaitAssertAsync(() => }, duration: TimeSpan.FromSeconds(10), cancellationToken: TestContext.Current.CancellationToken); } + [Fact] + public async Task PlainText_in_dm_flows_through_real_magic_byte_scanner() + { + // Regression: before the MagicByteValidator rewrite, only image MIMEs + // passed the scanner — text/plain was rejected with + // "File extension '.txt' is not allowed" even though the policy layer + // allowed it. This test uses the default (real) MagicByteContentScanner + // to prove the broader category support end-to-end. + _httpHandler.RespondWith("text/plain", FakePlainTextBytes); + var gateway = BuildGateway("slack-gw-plaintext-flow"); + + var files = new List + { + new("F_TXT", "notes.txt", "text/plain", FakePlainTextBytes.Length, + "https://files.slack.com/files-pri/T1234-F_TXT/notes.txt") + }; + + gateway.Tell(new SlackInboundMessage( + Kind: SlackInboundKind.Message, + EventId: new SlackEventId("D_TXT:3800"), + ChannelId: new SlackChannelId("D_TXT"), + ThreadTs: null, + EventTs: new SlackEventTs("3800.1"), + UserId: new SlackUserId("U_HUMAN"), + BotId: null, + Text: "here are my notes", + Subtype: null, + Hidden: false, + IsDirectMessage: true, + Files: files)); + + await AwaitAssertAsync(() => + { + Assert.Contains(_chatClient.ReceivedMessages, + contents => contents.Any(c => c is TextContent t + && t.Text.Contains("[attachment]", StringComparison.Ordinal) + && t.Text.Contains("notes.txt", StringComparison.Ordinal) + && t.Text.Contains("path=\"inbox/notes.txt\"", StringComparison.Ordinal))); + }, duration: TimeSpan.FromSeconds(10), cancellationToken: TestContext.Current.CancellationToken); + + var sessionId = new SessionId("D_TXT/3800.1"); + var inboxPath = Path.Combine( + SessionDirectoryHelper.GetOrCreateInboxDirectory(sessionId, _paths.SessionsDirectory), + "notes.txt"); + Assert.True(File.Exists(inboxPath), $"Expected inbox file at {inboxPath}"); + + // Scanner should not have posted any rejection reply. + Assert.DoesNotContain(_replyClient.PostedMessages, + m => m.Text.Contains("Content scanner rejected", StringComparison.Ordinal)); + } + [Fact] public async Task Scanner_rejection_surfaces_user_visible_reply_with_no_inbox_write() { diff --git a/src/Netclaw.Actors.Tests/Channels/SlackFileFlowIntegrationTests.cs b/src/Netclaw.Actors.Tests/Channels/SlackFileFlowIntegrationTests.cs index b49cb705c..a76699105 100644 --- a/src/Netclaw.Actors.Tests/Channels/SlackFileFlowIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Channels/SlackFileFlowIntegrationTests.cs @@ -1174,10 +1174,9 @@ await AwaitAssertAsync(() => } [Fact] - public async Task Scanner_failure_does_not_silently_drop_image() + public async Task Scanner_failure_rejects_attachment_and_does_not_inline() { - // When the content scanner itself is broken (e.g. TypeInitializationException), - // images should still flow through to the LLM rather than being silently dropped. + // Scanner failures must fail closed for inbound attachments. var pipeline = Host.Services.GetRequiredService(); var httpClient = new HttpClient(_httpHandler); @@ -1231,8 +1230,11 @@ await AwaitAssertAsync(() => "Expected at least one Slack reply to be posted"); }, duration: TimeSpan.FromSeconds(10), cancellationToken: TestContext.Current.CancellationToken); - Assert.True(_chatClient.ReceivedImageContent, - "Expected LLM to receive image even when scanner is broken"); + Assert.False(_chatClient.ReceivedImageContent, + "Expected LLM not to receive image when scanner fails"); + + Assert.Contains(_replyClient.PostedMessages, + m => m.Text.Contains("Couldn't scan `drawing.png`", StringComparison.Ordinal)); } /// diff --git a/src/Netclaw.Actors.Tests/Channels/SlackThreadBackfillIntegrationTests.cs b/src/Netclaw.Actors.Tests/Channels/SlackThreadBackfillIntegrationTests.cs index 288799792..d0a9acd66 100644 --- a/src/Netclaw.Actors.Tests/Channels/SlackThreadBackfillIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Channels/SlackThreadBackfillIntegrationTests.cs @@ -447,6 +447,107 @@ await AwaitAssertAsync(() => }, duration: TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); } + [Fact] + public async Task Backfill_document_in_public_channel_is_not_forwarded_as_data_content() + { + var pipeline = Host.Services.GetRequiredService(); + var httpClient = new HttpClient(_httpHandler); + + var fetcher = new SlackThreadHistoryFetcher( + (channelId, threadTs, limit, cursor, ct) => + Task.FromResult(new SlackNet.WebApi.ConversationMessagesResponse + { + Messages = + [ + new SlackNet.Events.MessageEvent + { + Ts = threadTs, + User = "U_ROOT", + Text = "thread root" + }, + new SlackNet.Events.MessageEvent + { + Ts = $"{threadTs[..^1]}1", + User = "U_ALICE", + Text = "historical doc", + Files = + [ + new SlackNet.File + { + Id = "F_DOCX", + Name = "notes.docx", + Mimetype = "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + Size = FakePngBytes.Length, + UrlPrivateDownload = "https://files.slack.com/fake/notes.docx" + } + ] + } + ] + }), + new SlackChannelOptions { BotToken = new SensitiveString("xoxb-fake") }, + httpClient, + new NullContentScanner(), + NullLogger.Instance); + + var profiles = ToolAudienceProfileDefaults.CreateProfiles(); + profiles.Public.ChannelAttachments = ToolAudienceProfileDefaults.CreatePublicChannelAttachments(); + + var deps = new SlackGatewayDependencies( + Pipeline: pipeline, + IngressGate: null, + ActorSystem: Sys, + TimeProvider: TimeProvider.System, + Options: new SlackChannelOptions + { + Enabled = true, + MentionOnly = true, + AllowedChannelIds = ["C_PUBLIC"], + ChannelAudiences = new Dictionary(StringComparer.Ordinal) + { + ["C_PUBLIC"] = "public" + }, + BotToken = new SensitiveString("xoxb-fake-token") + }, + BotUserId: new SlackUserId("UBOT"), + DefaultChannelId: null, + ReplyClient: _replyClient, + ContentScanner: new NullContentScanner(), + HttpClient: httpClient, + ThreadHistoryFetcher: fetcher, + AudienceProfiles: profiles, + ModelCapabilities: TestSlackGatewayDeps.DefaultVisionCapableModel, + Paths: _paths); + + var gateway = Sys.ActorOf(SlackGatewayActor.CreateProps(deps), "slack-gw-backfill-public-doc"); + + gateway.Tell(new SlackInboundMessage( + Kind: SlackInboundKind.AppMention, + EventId: new SlackEventId("C_PUBLIC:7000.2"), + ChannelId: new SlackChannelId("C_PUBLIC"), + ThreadTs: new SlackThreadTs("7000.0"), + EventTs: new SlackEventTs("7000.2"), + UserId: new SlackUserId("U_MENTIONER"), + BotId: null, + Text: "<@UBOT> please summarize", + Subtype: null, + Hidden: false, + IsDirectMessage: false)); + + await AwaitAssertAsync(() => + { + Assert.True(_chatClient.CallCount > 0, "Expected at least one LLM call"); + }, duration: TimeSpan.FromSeconds(10), cancellationToken: TestContext.Current.CancellationToken); + + var messages = _chatClient.LastMessages!; + var user = Assert.Single(messages, m => m.Role == AiChatRole.User); + + Assert.Empty(user.Contents.OfType()); + + var mergedText = string.Join("", user.Contents.OfType().Select(t => t.Text)); + Assert.Contains("attachment rejected", mergedText, StringComparison.OrdinalIgnoreCase); + Assert.Contains("category not allowed", mergedText, StringComparison.OrdinalIgnoreCase); + } + // --- Fake replies fetcher --- private Task FakeRepliesFetcher( diff --git a/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs b/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs index 8253db33a..c0bbd1a93 100644 --- a/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs +++ b/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs @@ -473,22 +473,18 @@ private async Task TryIngestSingleAttachmentAsync( if (!scanResult.IsAllowed) { + _log.Warning( + "slack_attachment_rejected name={Name} mime={Mime} reason=scan-blocked error={ScanError} message={ScanMessage}", + file.Name, file.MimeType, scanResult.Error?.ToString(), scanResult.Message ?? scanResult.Error?.ToString()); + if (scanResult.Error == ContentScanError.ScanFailure) { - // Scanner itself is broken — allow the file through rather than - // silently dropping it. The LLM provider will still validate. - _log.Error( - "Content scanner failed for file {Name}: {Message} — allowing file through", - file.Name, scanResult.Message); - } - else - { - _log.Warning( - "slack_attachment_rejected name={Name} mime={Mime} reason=scan-blocked message={ScanMessage}", - file.Name, file.MimeType, scanResult.Message ?? scanResult.Error?.ToString()); return new AttachmentIngestResult.Rejected( - $"Content scanner rejected `{file.Name}`: {scanResult.Message ?? scanResult.Error?.ToString()}."); + $"Couldn't scan `{file.Name}` — please try again later."); } + + return new AttachmentIngestResult.Rejected( + $"Content scanner rejected `{file.Name}`: {scanResult.Message ?? scanResult.Error?.ToString()}."); } // Write to inbox with filesystem-level collision suffixing and atomic move. @@ -794,7 +790,12 @@ private async Task BuildInputForInboundAsync( if (gap.Count == 0) return new InboundBuildResult(baseInput, detectorUnavailable); - var mergedContents = MergeGapWithLiveContents(gap, liveContents); + var mergedContents = MergeGapWithLiveContents( + gap, + liveContents, + triggeringMessage.Audience, + _dependencies.AudienceProfiles, + _dependencies.ModelCapabilities); return new InboundBuildResult(baseInput with { Contents = mergedContents }, detectorUnavailable); } @@ -875,12 +876,25 @@ private enum ClassificationOutcome private readonly record struct CursorAdvanced(string CursorTs); - private static List MergeGapWithLiveContents(IReadOnlyList gap, IReadOnlyList liveContents) + private static List MergeGapWithLiveContents( + IReadOnlyList gap, + IReadOnlyList liveContents, + TrustAudience audience, + ToolAudienceProfiles? audienceProfiles, + ModelCapabilities modelCapabilities) { + var profile = ToolAudienceProfileDefaults.GetResolvedProfile(audienceProfiles, audience); + var attachmentPolicy = profile.ChannelAttachments ?? ChannelAttachmentPolicy.Empty; + var inlineImages = modelCapabilities.InputModalities.HasFlag(ModelModality.Image); + var inlinePdfs = modelCapabilities.InputModalities.HasFlag(ModelModality.Image); + var sb = new StringBuilder(); sb.AppendLine("[thread history — messages exchanged before this inbound event]"); sb.AppendLine(); + var merged = new List(); + var acceptedBackfillData = new List(); + foreach (var item in gap) { var ts = item.ReceivedAt == default ? string.Empty : $", {item.ReceivedAt:yyyy-MM-dd HH:mm} UTC"; @@ -894,8 +908,36 @@ private static List MergeGapWithLiveContents(IReadOnlyList MergeGapWithLiveContents(IReadOnlyList { new TextContent(mergedText) }; - - // Gap image DataContent is carried through to the session so vision- - // capable models see the prior-thread images. The session actor's - // modality gate strips them when the resolved model doesn't support - // image input, so non-vision models are unaffected. - foreach (var item in gap) - { - foreach (var content in item.Contents) - { - if (content is not TextContent) - merged.Add(content); - } - } + merged.Add(new TextContent(mergedText)); + merged.AddRange(acceptedBackfillData); foreach (var content in liveContents) { diff --git a/src/Netclaw.Configuration.Tests/ChannelAttachmentPolicyTests.cs b/src/Netclaw.Configuration.Tests/ChannelAttachmentPolicyTests.cs index 594f56ce1..74c63c5df 100644 --- a/src/Netclaw.Configuration.Tests/ChannelAttachmentPolicyTests.cs +++ b/src/Netclaw.Configuration.Tests/ChannelAttachmentPolicyTests.cs @@ -134,7 +134,7 @@ public sealed class AttachmentCategoriesTests [InlineData("text/markdown", AttachmentCategory.Document)] [InlineData("application/json", AttachmentCategory.Document)] [InlineData("application/zip", AttachmentCategory.Archive)] - [InlineData("application/x-tar", AttachmentCategory.Archive)] + [InlineData("application/x-tar", AttachmentCategory.Other)] [InlineData("application/gzip", AttachmentCategory.Archive)] [InlineData("application/x-7z-compressed", AttachmentCategory.Archive)] [InlineData("video/mp4", AttachmentCategory.Media)] diff --git a/src/Netclaw.Configuration/ChannelAttachmentPolicy.cs b/src/Netclaw.Configuration/ChannelAttachmentPolicy.cs index 00913b65d..29ea0b7b2 100644 --- a/src/Netclaw.Configuration/ChannelAttachmentPolicy.cs +++ b/src/Netclaw.Configuration/ChannelAttachmentPolicy.cs @@ -105,12 +105,9 @@ public static AttachmentCategory FromMime(string? mime) { "application/zip" => true, "application/x-zip-compressed" => true, - "application/x-tar" => true, "application/gzip" => true, "application/x-gzip" => true, "application/x-7z-compressed" => true, - "application/x-rar-compressed" => true, - "application/vnd.rar" => true, "application/x-bzip2" => true, "application/x-xz" => true, _ => false diff --git a/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs b/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs index a371a6013..03255929a 100644 --- a/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs +++ b/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs @@ -1,30 +1,68 @@ +using System.Collections.Frozen; using Xunit; namespace Netclaw.Security.Tests; public sealed class MagicByteValidatorTests { - // Real PNG header bytes + // ── Image signatures ────────────────────────────────────────────────── private static readonly byte[] PngHeader = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00]; - - // Real JPEG header bytes private static readonly byte[] JpegHeader = [0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10]; - - // Real GIF header bytes (GIF89a) private static readonly byte[] GifHeader = [0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x01, 0x00, 0x01, 0x00]; - - // Real WebP header bytes (RIFF....WEBP) private static readonly byte[] WebpHeader = [0x52, 0x49, 0x46, 0x46, 0x24, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50, 0x56]; - // Windows EXE (MZ header) + // ── Document / PDF signatures ───────────────────────────────────────── + // %PDF-1.4 followed by a tiny catalog so the header has real-looking bytes + private static readonly byte[] PdfHeader = + "%PDF-1.4\n1 0 obj\n<< /Type /Catalog >>\nendobj\n"u8.ToArray(); + + // ZIP local file header (PK\x03\x04) — prefix of OOXML docx/xlsx/pptx and ZIP archives + private static readonly byte[] ZipHeader = + [0x50, 0x4B, 0x03, 0x04, 0x14, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00]; + + // OLE Compound Document — legacy .doc/.xls/.ppt + private static readonly byte[] OleHeader = + [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1, 0x00, 0x00]; + + // RTF header: {\rtf1\ansi... + private static readonly byte[] RtfHeader = "{\\rtf1\\ansi\\deff0"u8.ToArray(); + + private static readonly byte[] PlainTextBytes = "Hello, this is a plain text file with no magic signature.\n"u8.ToArray(); + + // ── Archive signatures ──────────────────────────────────────────────── + private static readonly byte[] SevenZipHeader = [0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C, 0x00, 0x04]; + private static readonly byte[] GzipHeader = [0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00]; + // bzip2: "BZh" + block size '9' + 6-byte BCD-Pi compressed block header (31 41 59 26 53 59) + private static readonly byte[] Bzip2Header = + [0x42, 0x5A, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59]; + private static readonly byte[] XzHeader = [0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00, 0x00, 0x04]; + + // ── Media signatures ────────────────────────────────────────────────── + // ISO BMFF: 4 bytes box size, "ftyp", brand, minor version, compatible brands + private static readonly byte[] Mp4FtypHeader = + [0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x6D, 0x70, 0x34, 0x32, 0x00, 0x00, 0x00, 0x00]; + // RIFF....WAVE + private static readonly byte[] WavHeader = + [0x52, 0x49, 0x46, 0x46, 0x24, 0x00, 0x00, 0x00, 0x57, 0x41, 0x56, 0x45, 0x66, 0x6D, 0x74, 0x20]; + // RIFF....AVI (trailing space) + private static readonly byte[] AviHeader = + [0x52, 0x49, 0x46, 0x46, 0x24, 0x00, 0x00, 0x00, 0x41, 0x56, 0x49, 0x20, 0x4C, 0x49, 0x53, 0x54]; + // ID3v2 tag prefix (used by most MP3s) + private static readonly byte[] Mp3Id3Header = [0x49, 0x44, 0x33, 0x03, 0x00, 0x00, 0x00, 0x00]; + // MP3 raw frame sync: 0xFF 0xFB MPEG-1 Layer 3 + private static readonly byte[] Mp3FrameHeader = [0xFF, 0xFB, 0x90, 0x00]; + // Ogg container + private static readonly byte[] OggHeader = [0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00]; + // Matroska / WebM EBML header + private static readonly byte[] EbmlHeader = [0x1A, 0x45, 0xDF, 0xA3, 0x9F, 0x42, 0x86, 0x81]; + + // ── Executable signatures ───────────────────────────────────────────── private static readonly byte[] ExeHeader = [0x4D, 0x5A, 0x90, 0x00, 0x03, 0x00, 0x00, 0x00]; - - // Linux ELF header private static readonly byte[] ElfHeader = [0x7F, 0x45, 0x4C, 0x46, 0x02, 0x01, 0x01, 0x00]; - - // Shebang script private static readonly byte[] ShebangHeader = [0x23, 0x21, 0x2F, 0x62, 0x69, 0x6E, 0x2F, 0x73, 0x68]; // #!/bin/sh + // ── Images ──────────────────────────────────────────────────────────── + [Fact] public void Validate_PngWithValidBytes_Allowed() { @@ -70,6 +108,311 @@ public void Validate_WebpWithValidBytes_Allowed() Assert.Null(result.Error); } + // ── PDF ─────────────────────────────────────────────────────────────── + + [Fact] + public void Validate_PdfWithValidMagicBytes_Allowed() + { + var result = MagicByteValidator.Validate(PdfHeader, "application/pdf", "report.pdf"); + + Assert.True(result.IsAllowed); + Assert.Null(result.Error); + } + + [Fact] + public void Validate_PdfExtensionWithPngPayload_MimeTypeMismatch() + { + var result = MagicByteValidator.Validate(PngHeader, "application/pdf", "fake.pdf"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_PdfPayloadDeclaredAsPng_MimeTypeMismatch() + { + var result = MagicByteValidator.Validate(PdfHeader, "image/png", "photo.png"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + // ── OOXML / OLE / ODF documents ─────────────────────────────────────── + + [Fact] + public void Validate_DocxWithZipMagic_Allowed() + { + var result = MagicByteValidator.Validate( + ZipHeader, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "report.docx"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_XlsxWithZipMagic_Allowed() + { + var result = MagicByteValidator.Validate( + ZipHeader, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "budget.xlsx"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_PptxWithZipMagic_Allowed() + { + var result = MagicByteValidator.Validate( + ZipHeader, + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "slides.pptx"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_OdtWithZipMagic_Allowed() + { + var result = MagicByteValidator.Validate( + ZipHeader, + "application/vnd.oasis.opendocument.text", + "notes.odt"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_DocxWithOleHeader_MimeTypeMismatch() + { + // Old .doc bytes declared as .docx (OOXML) — mismatch + var result = MagicByteValidator.Validate( + OleHeader, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "report.docx"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_LegacyDocWithOleHeader_Allowed() + { + var result = MagicByteValidator.Validate(OleHeader, "application/msword", "legacy.doc"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_LegacyXlsWithOleHeader_Allowed() + { + var result = MagicByteValidator.Validate(OleHeader, "application/vnd.ms-excel", "legacy.xls"); + + Assert.True(result.IsAllowed); + } + + // ── Plain / structured text ─────────────────────────────────────────── + + [Fact] + public void Validate_PlainTextWithoutMagic_Allowed() + { + var result = MagicByteValidator.Validate(PlainTextBytes, "text/plain", "notes.txt"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_MarkdownAllowed() + { + var result = MagicByteValidator.Validate(PlainTextBytes, "text/markdown", "readme.md"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_CsvAllowed() + { + var result = MagicByteValidator.Validate(PlainTextBytes, "text/csv", "data.csv"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_JsonAllowed() + { + var json = "{\"ok\":true}\n"u8.ToArray(); + var result = MagicByteValidator.Validate(json, "application/json", "payload.json"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_PlainTextWithExecutablePrefix_RejectedAsExecutable() + { + // Someone renames a Windows EXE to .txt — the executable pre-check + // fires regardless of declared MIME, protecting the text/plain path. + var result = MagicByteValidator.Validate(ExeHeader, "text/plain", "notes.txt"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.ExecutableContent, result.Error); + } + + [Fact] + public void Validate_RtfAllowed() + { + var result = MagicByteValidator.Validate(RtfHeader, "application/rtf", "document.rtf"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_RtfWithBogusMagic_MimeTypeMismatch() + { + var result = MagicByteValidator.Validate(PngHeader, "application/rtf", "fake.rtf"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + // ── Archives ────────────────────────────────────────────────────────── + + [Fact] + public void Validate_ZipAllowed() + { + var result = MagicByteValidator.Validate(ZipHeader, "application/zip", "archive.zip"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_SevenZipAllowed() + { + var result = MagicByteValidator.Validate(SevenZipHeader, "application/x-7z-compressed", "archive.7z"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_GzipAllowed() + { + var result = MagicByteValidator.Validate(GzipHeader, "application/gzip", "log.gz"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_Bzip2Allowed() + { + var result = MagicByteValidator.Validate(Bzip2Header, "application/x-bzip2", "backup.bz2"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_XzAllowed() + { + var result = MagicByteValidator.Validate(XzHeader, "application/x-xz", "backup.xz"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_ZipDeclaredAs7z_MimeTypeMismatch() + { + var result = MagicByteValidator.Validate(ZipHeader, "application/x-7z-compressed", "fake.7z"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + // ── Media ───────────────────────────────────────────────────────────── + + [Fact] + public void Validate_Mp4WithFtypBox_Allowed() + { + var result = MagicByteValidator.Validate(Mp4FtypHeader, "video/mp4", "clip.mp4"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_QuickTimeWithFtypBox_Allowed() + { + var result = MagicByteValidator.Validate(Mp4FtypHeader, "video/quicktime", "clip.mov"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_Mp3WithId3Tag_Allowed() + { + var result = MagicByteValidator.Validate(Mp3Id3Header, "audio/mpeg", "song.mp3"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_Mp3WithFrameSync_Allowed() + { + var result = MagicByteValidator.Validate(Mp3FrameHeader, "audio/mpeg", "song.mp3"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_WavAllowed() + { + var result = MagicByteValidator.Validate(WavHeader, "audio/wav", "sound.wav"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_AviAllowed() + { + var result = MagicByteValidator.Validate(AviHeader, "video/x-msvideo", "clip.avi"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_OggAllowed() + { + var result = MagicByteValidator.Validate(OggHeader, "audio/ogg", "sound.ogg"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_WebmAllowed() + { + var result = MagicByteValidator.Validate(EbmlHeader, "video/webm", "clip.webm"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_MatroskaAllowed() + { + var result = MagicByteValidator.Validate(EbmlHeader, "video/x-matroska", "clip.mkv"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_WavDeclaredAsMp4_MimeTypeMismatch() + { + // Both are "RIFF" but differ at offset 8 — WAV at offset 8 is "WAVE", + // MP4 at offset 4 is "ftyp". Make sure the stricter MP4 check rejects. + var result = MagicByteValidator.Validate(WavHeader, "video/mp4", "fake.mp4"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + // ── Cross-cutting ───────────────────────────────────────────────────── + [Fact] public void Validate_EmptyContent_Rejected() { @@ -108,11 +451,10 @@ public void Validate_ShebangScript_AlwaysRejected() } [Fact] - public void Validate_PdfMimeType_RejectedAsUnrecognized() + public void Validate_UnknownMimeType_Rejected() { - // PDF is not in image-only allowlist - byte[] pdfBytes = [0x25, 0x50, 0x44, 0x46, 0x2D, 0x31, 0x2E, 0x34]; // %PDF-1.4 - var result = MagicByteValidator.Validate(pdfBytes, "application/pdf", "doc.pdf"); + // application/octet-stream is classified as Other and not in the rule table + var result = MagicByteValidator.Validate(PngHeader, "application/octet-stream", "data.bin"); Assert.False(result.IsAllowed); Assert.Equal(ContentScanError.UnrecognizedFileType, result.Error); @@ -121,7 +463,8 @@ public void Validate_PdfMimeType_RejectedAsUnrecognized() [Fact] public void Validate_MimeTypeMismatch_ExtensionDoesNotMatchDeclaredType() { - // PNG bytes but declared as JPEG + // PNG bytes but declared as JPEG → mismatch (rule lookup finds jpeg, + // extension .png is not in jpeg's {.jpg,.jpeg} set) var result = MagicByteValidator.Validate(PngHeader, "image/jpeg", "photo.png"); Assert.False(result.IsAllowed); @@ -139,10 +482,30 @@ public void Validate_MagicByteMismatch_JpegBytesWithPngDeclaration() } [Fact] - public void Validate_UnknownExtension_Rejected() + public void Validate_UnknownExtensionForKnownMime_MimeTypeMismatch() { + // PNG bytes with MIME image/png, but filename is .bmp — extension + // is not in image/png's extension set, so the rule rejects it + // as a mismatch (not UnrecognizedFileType — the MIME is known). var result = MagicByteValidator.Validate(PngHeader, "image/png", "photo.bmp"); + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_PolicyDisallowsKnownMime_Rejected() + { + // Operator restricts policy to PNG only; PDF is known to the validator + // but disallowed by the runtime policy layer. + var policy = new ContentPolicy + { + AllowedMimeTypes = new HashSet(StringComparer.OrdinalIgnoreCase) + { "image/png" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase) + }; + + var result = MagicByteValidator.Validate(PdfHeader, "application/pdf", "report.pdf", policy); + Assert.False(result.IsAllowed); Assert.Equal(ContentScanError.UnrecognizedFileType, result.Error); } @@ -165,6 +528,7 @@ public void HasExecutableSignature_DetectsAllKnownSignatures() Assert.True(MagicByteValidator.HasExecutableSignature(ShebangHeader)); Assert.False(MagicByteValidator.HasExecutableSignature(PngHeader)); Assert.False(MagicByteValidator.HasExecutableSignature(JpegHeader)); + Assert.False(MagicByteValidator.HasExecutableSignature(PdfHeader)); } [Theory] @@ -172,7 +536,16 @@ public void HasExecutableSignature_DetectsAllKnownSignatures() [InlineData(nameof(JpegHeader), "image/jpeg")] [InlineData(nameof(GifHeader), "image/gif")] [InlineData(nameof(WebpHeader), "image/webp")] - public void DetectMimeType_identifies_supported_image_types(string headerField, string expectedMime) + [InlineData(nameof(PdfHeader), "application/pdf")] + [InlineData(nameof(ZipHeader), "application/zip")] + [InlineData(nameof(OleHeader), "application/x-ole-compound-document")] + [InlineData(nameof(Mp4FtypHeader), "video/mp4")] + [InlineData(nameof(WavHeader), "audio/wav")] + [InlineData(nameof(AviHeader), "video/x-msvideo")] + [InlineData(nameof(EbmlHeader), "video/webm")] + [InlineData(nameof(OggHeader), "audio/ogg")] + [InlineData(nameof(Mp3Id3Header), "audio/mpeg")] + public void DetectMimeType_identifies_supported_signature_families(string headerField, string expectedMime) { var header = headerField switch { @@ -180,12 +553,167 @@ public void DetectMimeType_identifies_supported_image_types(string headerField, nameof(JpegHeader) => JpegHeader, nameof(GifHeader) => GifHeader, nameof(WebpHeader) => WebpHeader, + nameof(PdfHeader) => PdfHeader, + nameof(ZipHeader) => ZipHeader, + nameof(OleHeader) => OleHeader, + nameof(Mp4FtypHeader) => Mp4FtypHeader, + nameof(WavHeader) => WavHeader, + nameof(AviHeader) => AviHeader, + nameof(EbmlHeader) => EbmlHeader, + nameof(OggHeader) => OggHeader, + nameof(Mp3Id3Header) => Mp3Id3Header, _ => throw new ArgumentException(headerField) }; Assert.Equal(expectedMime, MagicByteValidator.DetectMimeType(header)); } + // ── Hardening: reject minimum-magic polyglots ───────────────────────── + + [Fact] + public void Validate_RejectsJpegWithStuffedMarker() + { + // FF D8 FF 00 — FF 00 is an escaped stuffed byte inside JPEG data, + // not a valid JPEG start-of-image marker. A minimum-magic checker + // would accept this as JPEG; the hardened check rejects it. + byte[] bogusJpeg = [0xFF, 0xD8, 0xFF, 0x00, 0x10, 0x4A]; + var result = MagicByteValidator.Validate(bogusJpeg, "image/jpeg", "bogus.jpg"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsGifWithBogusVersion() + { + // GIF8Xa — minimum check (GIF8 prefix) would accept; tightened check + // requires '7a' or '9a' at bytes 4-5. + byte[] bogusGif = [0x47, 0x49, 0x46, 0x38, 0x58, 0x61, 0x01]; + var result = MagicByteValidator.Validate(bogusGif, "image/gif", "bogus.gif"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsPdfWithoutVersionDigit() + { + // %PDF-X.0 — missing digit after the dash. Minimum 5-byte check + // would accept; tightened check requires a digit + dot. + byte[] bogusPdf = "%PDF-X.0\n"u8.ToArray(); + var result = MagicByteValidator.Validate(bogusPdf, "application/pdf", "bogus.pdf"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsRtfWithoutVersionDigit() + { + // {\rtfX — RTF spec requires a version digit after \rtf. + byte[] bogusRtf = "{\\rtfX junk"u8.ToArray(); + var result = MagicByteValidator.Validate(bogusRtf, "application/rtf", "bogus.rtf"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsZipWithInvalidHeaderPair() + { + // PK\x03\x06 is not a valid ZIP local/central-dir/spanned marker. + // Minimum check (PK + any of 03/05/07 + any of 04/06/08) would have + // accepted this; tightened check requires the exact 4-byte pair. + byte[] bogusZip = [0x50, 0x4B, 0x03, 0x06, 0x14, 0x00, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusZip, "application/zip", "bogus.zip"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsGzipWithNonDeflateMethod() + { + // 1F 8B 01 — compression method 1 is reserved; only 0x08 (DEFLATE) + // is defined by RFC 1952. Minimum 2-byte check would accept. + byte[] bogusGzip = [0x1F, 0x8B, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusGzip, "application/gzip", "bogus.gz"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsBzip2WithBogusBlockHeader() + { + // BZh9 + garbage where the BCD-Pi block header should be. + byte[] bogusBzip2 = [0x42, 0x5A, 0x68, 0x39, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]; + var result = MagicByteValidator.Validate(bogusBzip2, "application/x-bzip2", "bogus.bz2"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsFtypWithNonAsciiMajorBrand() + { + // Valid "ftyp" box header but major brand bytes are not printable + // ASCII — indicates a garbage or adversarial box payload. + byte[] bogusFtyp = + [0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x00, 0x01, 0x02, 0x03]; + var result = MagicByteValidator.Validate(bogusFtyp, "video/mp4", "bogus.mp4"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsOggWithNonZeroVersion() + { + // OggS + version byte 0x01 — RFC 3533 requires version 0x00. + byte[] bogusOgg = [0x4F, 0x67, 0x67, 0x53, 0x01, 0x02, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusOgg, "audio/ogg", "bogus.ogg"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsMp3WithReservedLayer() + { + // 0xFF 0xF9 — top nibble F (12-bit sync OK), version bits 11 + // (MPEG-1 OK), but layer bits (2-1) = 00 → reserved per ISO/IEC + // 11172-3. 0xF9 = 1111 1001, bits 2-1 = 00. + byte[] bogusMp3 = [0xFF, 0xF9, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusMp3, "audio/mpeg", "bogus.mp3"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsMp3With11BitSyncPolyglot() + { + // 0xFF 0xE0 — only 11-bit sync. The looser check accepted this; + // the tightened 12-bit sync requires top nibble F, which rejects. + byte[] bogusMp3 = [0xFF, 0xE0, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusMp3, "audio/mpeg", "bogus.mp3"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsId3WithBogusMajorVersion() + { + // ID3 + major version 99 — valid ID3v2 is 2, 3, or 4. + byte[] bogusId3 = [0x49, 0x44, 0x33, 0x63, 0x00, 0x00, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusId3, "audio/mpeg", "bogus.mp3"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + [Fact] public void DetectMimeType_returns_null_for_unknown_content() { diff --git a/src/Netclaw.Security/ContentPolicy.cs b/src/Netclaw.Security/ContentPolicy.cs index 92b0fda73..348e72638 100644 --- a/src/Netclaw.Security/ContentPolicy.cs +++ b/src/Netclaw.Security/ContentPolicy.cs @@ -3,18 +3,26 @@ namespace Netclaw.Security; /// -/// Configurable content security policy for file uploads. +/// Configurable content security policy for file uploads. Layered on top of +/// : the validator advertises which MIME +/// types it knows how to verify, and this policy is the runtime allowlist +/// that operators can use to further restrict what passes. /// public sealed class ContentPolicy { /// - /// Default maximum file size: 20 MB. + /// Default maximum file size: 25 MiB. Tracks + /// ChannelAttachmentPolicy.DefaultMaxFileBytes so the scanner + /// ceiling and the channel policy ceiling don't drift. /// - public const long DefaultMaxFileSizeBytes = 20 * 1024 * 1024; + public const long DefaultMaxFileSizeBytes = 25L * 1024 * 1024; /// - /// MIME types allowed through the content scanner. - /// Defaults to image types supported by vision models. + /// MIME types allowed through the content scanner. Defaults to the full + /// set of MIME types knows how to verify + /// — images, PDF, Office documents, plain/rich text, archives, and + /// common audio/video containers. Operators can override this to + /// restrict the allowlist further. /// public FrozenSet AllowedMimeTypes { get; init; } = DefaultAllowedMimeTypes; @@ -24,11 +32,6 @@ public sealed class ContentPolicy public long MaxFileSizeBytes { get; init; } = DefaultMaxFileSizeBytes; public static readonly FrozenSet DefaultAllowedMimeTypes = - new HashSet(StringComparer.OrdinalIgnoreCase) - { - "image/png", - "image/jpeg", - "image/gif", - "image/webp" - }.ToFrozenSet(StringComparer.OrdinalIgnoreCase); + MagicByteValidator.GetSupportedMimeTypes() + .ToFrozenSet(StringComparer.OrdinalIgnoreCase); } diff --git a/src/Netclaw.Security/MagicByteValidator.cs b/src/Netclaw.Security/MagicByteValidator.cs index 0431b7d62..0c8ffd866 100644 --- a/src/Netclaw.Security/MagicByteValidator.cs +++ b/src/Netclaw.Security/MagicByteValidator.cs @@ -4,31 +4,17 @@ namespace Netclaw.Security; /// /// Validates file content using magic byte (file signature) analysis. -/// Narrowed to image types for Netclaw's multimodal pipeline. +/// Supports the categories advertised by ChannelAttachmentPolicy's +/// Team / Personal audience defaults: images, PDFs, Office +/// documents (OOXML + legacy OLE), plain/rich text, common archives, and +/// common audio/video containers. Unknown declared MIME types are rejected +/// as . /// public static class MagicByteValidator { - private static readonly FrozenSet AllowedImageMimeTypes = - new HashSet(StringComparer.OrdinalIgnoreCase) - { - "image/png", - "image/jpeg", - "image/gif", - "image/webp" - }.ToFrozenSet(StringComparer.OrdinalIgnoreCase); - - private static readonly FrozenDictionary> AllowedExtensions = - new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - [".png"] = new[] { "image/png" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase), - [".jpg"] = new[] { "image/jpeg" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase), - [".jpeg"] = new[] { "image/jpeg" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase), - [".gif"] = new[] { "image/gif" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase), - [".webp"] = new[] { "image/webp" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase), - }.ToFrozenDictionary(StringComparer.OrdinalIgnoreCase); - /// - /// Magic byte signatures for executable content — always blocked. + /// Magic byte signatures for executable content — always blocked, + /// regardless of declared MIME type. /// private static readonly byte[][] ExecutableSignatures = [ @@ -40,9 +26,39 @@ public static class MagicByteValidator [0x23, 0x21] // #! — Shebang (shell scripts) ]; + /// + /// Matcher delegate for a signature rule. Uses a typed delegate instead + /// of Func<ReadOnlySpan<byte>, bool> because + /// cannot be a generic type argument. + /// + private delegate bool SignatureMatcher(ReadOnlySpan content); + + /// + /// Signature rule for a declared MIME type. Extensions are the set of + /// filename extensions permitted to carry this MIME; + /// returns true when the raw bytes satisfy the signature family. + /// + private sealed record SignatureRule( + FrozenSet Extensions, + SignatureMatcher Matches); + + /// + /// Matcher that unconditionally accepts any content. Used for text-like + /// MIMEs that have no signature at offset 0; the executable pre-check + /// still runs first, so an MZ- or #!-prefixed "text" file + /// is still rejected as . + /// + private static readonly SignatureMatcher AnyContent = static _ => true; + + private static readonly FrozenDictionary RulesByMime = + BuildRules().ToFrozenDictionary(StringComparer.OrdinalIgnoreCase); + /// /// Validates file content against its declared MIME type and filename. - /// Only image types are allowed through. + /// Rejects empty content, oversized content, executables, unknown or + /// disallowed MIME types, filenames whose extension doesn't match the + /// declared MIME type, and content whose magic bytes don't match the + /// declared signature family. /// public static ContentScanResult Validate( ReadOnlySpan content, @@ -58,16 +74,14 @@ public static ContentScanResult Validate( } var effectivePolicy = policy ?? new ContentPolicy(); - var allowedMimes = effectivePolicy.AllowedMimeTypes; if (content.Length > effectivePolicy.MaxFileSizeBytes) { return ContentScanResult.Rejected( ContentScanError.FileTooLarge, - $"File exceeds maximum size of {effectivePolicy.MaxFileSizeBytes / (1024 * 1024)} MB"); + $"File exceeds maximum size of {effectivePolicy.MaxFileSizeBytes / (1024 * 1024)} MiB"); } - // Always check for executables — these are never allowed if (HasExecutableSignature(content)) { var detectedType = DetectMimeType(content); @@ -77,65 +91,73 @@ public static ContentScanResult Validate( detectedType is not null ? new MimeType(detectedType) : null); } - // Validate extension is in allowlist - var extension = Path.GetExtension(filename); - if (string.IsNullOrEmpty(extension) || !AllowedExtensions.ContainsKey(extension)) + if (!RulesByMime.TryGetValue(declaredMimeType, out var rule)) { return ContentScanResult.Rejected( ContentScanError.UnrecognizedFileType, - $"File extension '{extension}' is not allowed"); + $"MIME type '{declaredMimeType}' is not supported by the content scanner"); } - // Validate declared MIME type is allowed - if (!allowedMimes.Contains(declaredMimeType)) + if (!effectivePolicy.AllowedMimeTypes.Contains(declaredMimeType)) { return ContentScanResult.Rejected( ContentScanError.UnrecognizedFileType, - $"MIME type '{declaredMimeType}' is not allowed"); + $"MIME type '{declaredMimeType}' is not allowed by policy"); } - // Validate extension matches declared MIME type - var allowedMimesForExtension = AllowedExtensions[extension]; - if (!allowedMimesForExtension.Contains(declaredMimeType)) + var extension = Path.GetExtension(filename); + if (string.IsNullOrEmpty(extension)) + { + return ContentScanResult.Rejected( + ContentScanError.UnrecognizedFileType, + "File has no extension"); + } + + if (!rule.Extensions.Contains(extension)) { return ContentScanResult.Rejected( ContentScanError.MimeTypeMismatch, $"Extension '{extension}' does not match declared type '{declaredMimeType}'"); } - // Validate magic bytes match expected image type - return ValidateImageContent(content, declaredMimeType); + if (!rule.Matches(content)) + { + var detectedMimeType = DetectMimeType(content); + return ContentScanResult.Rejected( + ContentScanError.MimeTypeMismatch, + $"Content is not a valid {declaredMimeType} file", + detectedMimeType is not null ? new MimeType(detectedMimeType) : null); + } + + return ContentScanResult.Allowed(new MimeType(declaredMimeType)); } /// - /// Detects MIME type from magic bytes for the supported image types. + /// Detects MIME type from magic bytes for supported signature families. + /// Used to populate the DetectedType field on rejection results so + /// operators can see "looks like a PDF but was declared as an image". /// Returns null for unrecognized content. /// public static string? DetectMimeType(ReadOnlySpan content) { - if (content.Length >= 8 && - content[0] == 0x89 && content[1] == 0x50 && - content[2] == 0x4E && content[3] == 0x47 && - content[4] == 0x0D && content[5] == 0x0A && - content[6] == 0x1A && content[7] == 0x0A) - return "image/png"; - - if (content.Length >= 3 && - content[0] == 0xFF && content[1] == 0xD8 && content[2] == 0xFF) - return "image/jpeg"; - - if (content.Length >= 4 && - content[0] == 0x47 && content[1] == 0x49 && - content[2] == 0x46 && content[3] == 0x38) - return "image/gif"; - - if (content.Length >= 12 && - content[0] == 0x52 && content[1] == 0x49 && - content[2] == 0x46 && content[3] == 0x46 && - content[8] == 0x57 && content[9] == 0x45 && - content[10] == 0x42 && content[11] == 0x50) - return "image/webp"; - + if (IsPng(content)) return "image/png"; + if (IsJpeg(content)) return "image/jpeg"; + if (IsGif(content)) return "image/gif"; + if (IsWebp(content)) return "image/webp"; + if (IsPdf(content)) return "application/pdf"; + if (IsOle(content)) return "application/x-ole-compound-document"; + if (IsRtf(content)) return "application/rtf"; + if (Is7z(content)) return "application/x-7z-compressed"; + if (IsGzip(content)) return "application/gzip"; + if (IsBzip2(content)) return "application/x-bzip2"; + if (IsXz(content)) return "application/x-xz"; + if (IsZip(content)) return "application/zip"; + if (IsWav(content)) return "audio/wav"; + if (IsAvi(content)) return "video/x-msvideo"; + if (IsFtyp(content)) return "video/mp4"; + if (IsEbml(content)) return "video/webm"; + if (IsOgg(content)) return "audio/ogg"; + if (IsMp3FrameOrId3(content)) return "audio/mpeg"; return null; } @@ -153,69 +175,290 @@ public static bool HasExecutableSignature(ReadOnlySpan content) return false; } - private static ContentScanResult ValidateImageContent( - ReadOnlySpan content, - string declaredMimeType) + // ── Rule table ──────────────────────────────────────────────────────── + + private static Dictionary BuildRules() { - var detectedMimeType = DetectMimeType(content); + var rules = new Dictionary(StringComparer.OrdinalIgnoreCase); - if (declaredMimeType.Equals("image/png", StringComparison.OrdinalIgnoreCase)) - { - // PNG: 89 50 4E 47 0D 0A 1A 0A - if (content.Length < 8 || - content[0] != 0x89 || content[1] != 0x50 || - content[2] != 0x4E || content[3] != 0x47 || - content[4] != 0x0D || content[5] != 0x0A || - content[6] != 0x1A || content[7] != 0x0A) - { - return ContentScanResult.Rejected( - ContentScanError.MimeTypeMismatch, - "Content is not a valid PNG file", - detectedMimeType is not null ? new MimeType(detectedMimeType) : null); - } - } - else if (declaredMimeType.Equals("image/jpeg", StringComparison.OrdinalIgnoreCase)) - { - // JPEG: FF D8 FF - if (content.Length < 3 || - content[0] != 0xFF || content[1] != 0xD8 || content[2] != 0xFF) - { - return ContentScanResult.Rejected( - ContentScanError.MimeTypeMismatch, - "Content is not a valid JPEG file", - detectedMimeType is not null ? new MimeType(detectedMimeType) : null); - } - } - else if (declaredMimeType.Equals("image/gif", StringComparison.OrdinalIgnoreCase)) + // Images + rules["image/png"] = new(Exts(".png"), IsPng); + rules["image/jpeg"] = new(Exts(".jpg", ".jpeg"), IsJpeg); + rules["image/gif"] = new(Exts(".gif"), IsGif); + rules["image/webp"] = new(Exts(".webp"), IsWebp); + + // PDF + rules["application/pdf"] = new(Exts(".pdf"), IsPdf); + + // OOXML (ZIP-based) — docx/xlsx/pptx + rules["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = + new(Exts(".docx"), IsZip); + rules["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = + new(Exts(".xlsx"), IsZip); + rules["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = + new(Exts(".pptx"), IsZip); + + // OpenDocument (also ZIP-based) + rules["application/vnd.oasis.opendocument.text"] = new(Exts(".odt"), IsZip); + rules["application/vnd.oasis.opendocument.spreadsheet"] = new(Exts(".ods"), IsZip); + rules["application/vnd.oasis.opendocument.presentation"] = new(Exts(".odp"), IsZip); + + // Legacy OLE Compound Document Office formats + rules["application/msword"] = new(Exts(".doc"), IsOle); + rules["application/vnd.ms-excel"] = new(Exts(".xls"), IsOle); + rules["application/vnd.ms-powerpoint"] = new(Exts(".ppt"), IsOle); + + // Plain/structured text — no signature, but executable pre-check + // still blocks MZ / ELF / shebang payloads. + rules["text/plain"] = new(Exts(".txt", ".log"), AnyContent); + rules["text/markdown"] = new(Exts(".md", ".markdown"), AnyContent); + rules["text/csv"] = new(Exts(".csv"), AnyContent); + rules["text/xml"] = new(Exts(".xml"), AnyContent); + rules["application/json"] = new(Exts(".json"), AnyContent); + rules["application/xml"] = new(Exts(".xml"), AnyContent); + rules["application/yaml"] = new(Exts(".yml", ".yaml"), AnyContent); + rules["application/x-yaml"] = new(Exts(".yml", ".yaml"), AnyContent); + + // Rich text + rules["application/rtf"] = new(Exts(".rtf"), IsRtf); + rules["text/rtf"] = new(Exts(".rtf"), IsRtf); + + // Archives + rules["application/zip"] = new(Exts(".zip"), IsZip); + rules["application/x-zip-compressed"] = new(Exts(".zip"), IsZip); + rules["application/x-7z-compressed"] = new(Exts(".7z"), Is7z); + rules["application/gzip"] = new(Exts(".gz", ".tgz"), IsGzip); + rules["application/x-gzip"] = new(Exts(".gz", ".tgz"), IsGzip); + rules["application/x-bzip2"] = new(Exts(".bz2"), IsBzip2); + rules["application/x-xz"] = new(Exts(".xz"), IsXz); + + // Audio + rules["audio/mpeg"] = new(Exts(".mp3"), IsMp3FrameOrId3); + rules["audio/mp4"] = new(Exts(".m4a", ".mp4"), IsFtyp); + rules["audio/x-m4a"] = new(Exts(".m4a"), IsFtyp); + rules["audio/wav"] = new(Exts(".wav"), IsWav); + rules["audio/x-wav"] = new(Exts(".wav"), IsWav); + rules["audio/ogg"] = new(Exts(".ogg", ".oga"), IsOgg); + + // Video + rules["video/mp4"] = new(Exts(".mp4", ".m4v"), IsFtyp); + rules["video/quicktime"] = new(Exts(".mov"), IsFtyp); + rules["video/webm"] = new(Exts(".webm"), IsEbml); + rules["video/x-matroska"] = new(Exts(".mkv"), IsEbml); + rules["video/x-msvideo"] = new(Exts(".avi"), IsAvi); + + return rules; + } + + private static FrozenSet Exts(params string[] extensions) => + extensions.ToFrozenSet(StringComparer.OrdinalIgnoreCase); + + /// + /// Internal helper exposed for to seed its + /// default allowlist from the set of MIME types the validator knows how + /// to verify, so the two layers stay in sync. + /// + internal static IEnumerable GetSupportedMimeTypes() => RulesByMime.Keys; + + // ── Signature matchers ──────────────────────────────────────────────── + // Each matcher validates more than the minimum prefix where a cheap + // follow-on check is available. The goal is to reject polyglots and + // hand-forged headers that match the minimum magic but have bogus + // version/layer/brand fields, without false-rejecting real files. + + private static bool IsPng(ReadOnlySpan c) => + c.Length >= 8 && + c[0] == 0x89 && c[1] == 0x50 && c[2] == 0x4E && c[3] == 0x47 && + c[4] == 0x0D && c[5] == 0x0A && c[6] == 0x1A && c[7] == 0x0A; + + /// + /// JPEG: SOI (FF D8) followed by a marker byte. The third byte + /// must look like a JPEG marker (high nibble F, low nibble not + /// 0 or F — FF 00 is a stuffed byte inside data, + /// FF FF is fill). + /// + private static bool IsJpeg(ReadOnlySpan c) + { + if (c.Length < 3) return false; + if (c[0] != 0xFF || c[1] != 0xD8 || c[2] != 0xFF) return false; + if (c.Length < 4) return false; + var marker = c[3]; + return marker != 0x00 && marker != 0xFF; + } + + /// + /// GIF: GIF87a or GIF89a — validate all 6 bytes rather + /// than just the GIF8 prefix so GIF8Xa prefixes from + /// adversarial content are rejected. + /// + private static bool IsGif(ReadOnlySpan c) + { + if (c.Length < 6) return false; + if (c[0] != 0x47 || c[1] != 0x49 || c[2] != 0x46 || c[3] != 0x38) return false; + if (c[5] != 0x61) return false; // 'a' + return c[4] == 0x37 || c[4] == 0x39; // '7' or '9' + } + + private static bool IsRiff(ReadOnlySpan c) => + c.Length >= 4 && + c[0] == 0x52 && c[1] == 0x49 && c[2] == 0x46 && c[3] == 0x46; + + private static bool IsWebp(ReadOnlySpan c) => + IsRiff(c) && c.Length >= 12 && + c[8] == 0x57 && c[9] == 0x45 && c[10] == 0x42 && c[11] == 0x50; + + private static bool IsWav(ReadOnlySpan c) => + IsRiff(c) && c.Length >= 12 && + c[8] == 0x57 && c[9] == 0x41 && c[10] == 0x56 && c[11] == 0x45; + + private static bool IsAvi(ReadOnlySpan c) => + IsRiff(c) && c.Length >= 12 && + c[8] == 0x41 && c[9] == 0x56 && c[10] == 0x49 && c[11] == 0x20; + + /// + /// PDF: %PDF- followed by a version digit and a dot. A minimal + /// valid PDF starts with %PDF-1.0 through %PDF-2.0; the + /// tightened check requires byte 5 to be an ASCII digit and byte 6 to + /// be a dot, rejecting %PDF-xx garbage headers. + /// + private static bool IsPdf(ReadOnlySpan c) + { + if (c.Length < 7) return false; + if (c[0] != 0x25 || c[1] != 0x50 || c[2] != 0x44 || c[3] != 0x46 || c[4] != 0x2D) + return false; + return c[5] is >= (byte)'0' and <= (byte)'9' && c[6] == (byte)'.'; + } + + /// + /// ZIP local file header (PK\x03\x04), central directory end + /// (PK\x05\x06), or spanned-archive marker (PK\x07\x08). + /// The third and fourth bytes must form one of those three exact pairs + /// to reject PK\x03\x06-style garbage. + /// + private static bool IsZip(ReadOnlySpan c) + { + if (c.Length < 4) return false; + if (c[0] != 0x50 || c[1] != 0x4B) return false; + return (c[2] == 0x03 && c[3] == 0x04) + || (c[2] == 0x05 && c[3] == 0x06) + || (c[2] == 0x07 && c[3] == 0x08); + } + + private static bool IsOle(ReadOnlySpan c) => + c.Length >= 8 && + c[0] == 0xD0 && c[1] == 0xCF && c[2] == 0x11 && c[3] == 0xE0 && + c[4] == 0xA1 && c[5] == 0xB1 && c[6] == 0x1A && c[7] == 0xE1; + + /// + /// RTF: {\rtf followed by a version digit (RTF spec requires + /// {\rtf1 — the 1 in practice, but the spec allows any + /// digit for the version field). + /// + private static bool IsRtf(ReadOnlySpan c) + { + if (c.Length < 6) return false; + if (c[0] != 0x7B || c[1] != 0x5C || c[2] != 0x72 || c[3] != 0x74 || c[4] != 0x66) + return false; + return c[5] is >= (byte)'0' and <= (byte)'9'; + } + + private static bool Is7z(ReadOnlySpan c) => + c.Length >= 6 && + c[0] == 0x37 && c[1] == 0x7A && c[2] == 0xBC && c[3] == 0xAF && + c[4] == 0x27 && c[5] == 0x1C; + + /// + /// Gzip: 1F 8B plus compression method 08 (DEFLATE). + /// RFC 1952 reserves methods 0–7; only 8 is defined. Rejecting + /// unknown methods blocks crafted gzip polyglots. + /// + private static bool IsGzip(ReadOnlySpan c) => + c.Length >= 3 && c[0] == 0x1F && c[1] == 0x8B && c[2] == 0x08; + + /// + /// Bzip2: BZh + block size digit (1–9, hundreds of + /// kilobytes) + the 6-byte BCD-encoded magic number 31 41 59 26 53 59 + /// (the first 6 digits of Pi — the bzip2 compressed-block header). + /// Validates full 10-byte prefix, not just BZh. + /// + private static bool IsBzip2(ReadOnlySpan c) + { + if (c.Length < 10) return false; + if (c[0] != 0x42 || c[1] != 0x5A || c[2] != 0x68) return false; + if (c[3] is < (byte)'1' or > (byte)'9') return false; + return c[4] == 0x31 && c[5] == 0x41 && c[6] == 0x59 + && c[7] == 0x26 && c[8] == 0x53 && c[9] == 0x59; + } + + private static bool IsXz(ReadOnlySpan c) => + c.Length >= 6 && + c[0] == 0xFD && c[1] == 0x37 && c[2] == 0x7A && c[3] == 0x58 && + c[4] == 0x5A && c[5] == 0x00; + + /// + /// ISO BMFF: 4-byte box size (≥ 8), ftyp at offset 4, then a + /// 4-byte major brand which must be printable ASCII. Rejects headers + /// where the major brand is garbage — a common polyglot failure mode. + /// + private static bool IsFtyp(ReadOnlySpan c) + { + if (c.Length < 12) return false; + if (c[4] != 0x66 || c[5] != 0x74 || c[6] != 0x79 || c[7] != 0x70) return false; + + // Box size big-endian uint32 at bytes 0-3. Must be ≥ 8 and either + // representable (≤ content length + some slack) or the special + // value 0 (box extends to end-of-file) or 1 (64-bit size follows). + uint boxSize = ((uint)c[0] << 24) | ((uint)c[1] << 16) | ((uint)c[2] << 8) | c[3]; + if (boxSize != 0 && boxSize != 1 && boxSize < 8) return false; + + // Major brand at bytes 8-11 must be printable ASCII. + for (var i = 8; i < 12; i++) { - // GIF: GIF8 (47 49 46 38) - if (content.Length < 4 || - content[0] != 0x47 || content[1] != 0x49 || - content[2] != 0x46 || content[3] != 0x38) - { - return ContentScanResult.Rejected( - ContentScanError.MimeTypeMismatch, - "Content is not a valid GIF file", - detectedMimeType is not null ? new MimeType(detectedMimeType) : null); - } + var b = c[i]; + if (b < 0x20 || b > 0x7E) return false; } - else if (declaredMimeType.Equals("image/webp", StringComparison.OrdinalIgnoreCase)) + return true; + } + + private static bool IsEbml(ReadOnlySpan c) => + c.Length >= 4 && + c[0] == 0x1A && c[1] == 0x45 && c[2] == 0xDF && c[3] == 0xA3; + + /// + /// Ogg: OggS followed by a version byte that per RFC 3533 §6 + /// must be 0x00. Rejects Ogg-prefixed polyglots with bogus version. + /// + private static bool IsOgg(ReadOnlySpan c) => + c.Length >= 5 && + c[0] == 0x4F && c[1] == 0x67 && c[2] == 0x67 && c[3] == 0x53 && + c[4] == 0x00; + + /// + /// MP3: either an ID3v2 tag (ID3 + valid major version 2/3/4) + /// or a raw MPEG audio frame sync. Uses the strict 12-bit sync + /// (FF Fx) and validates the layer bits are not reserved. + /// 12-bit sync is stricter than the 11-bit form — it rejects + /// MPEG-2.5 (rare/non-standard) and FF E* polyglots — and + /// implicitly rules out reserved MPEG version 01, since that would + /// require bit 4 = 0 (top nibble E, not F). Layer bits + /// (bits 2–1 of byte 1) are still checked: 00 is reserved + /// per ISO/IEC 11172-3 and must be rejected. + /// + private static bool IsMp3FrameOrId3(ReadOnlySpan c) + { + if (c.Length >= 4 && c[0] == 0x49 && c[1] == 0x44 && c[2] == 0x33) { - // WebP: RIFF....WEBP (52 49 46 46 xx xx xx xx 57 45 42 50) - if (content.Length < 12 || - content[0] != 0x52 || content[1] != 0x49 || - content[2] != 0x46 || content[3] != 0x46 || - content[8] != 0x57 || content[9] != 0x45 || - content[10] != 0x42 || content[11] != 0x50) - { - return ContentScanResult.Rejected( - ContentScanError.MimeTypeMismatch, - "Content is not a valid WebP file", - detectedMimeType is not null ? new MimeType(detectedMimeType) : null); - } + // ID3v2 major version (byte 3) is 2, 3, or 4 in the wild. + return c[3] is 0x02 or 0x03 or 0x04; } - return ContentScanResult.Allowed( - new MimeType(detectedMimeType ?? declaredMimeType)); + if (c.Length < 2) return false; + if (c[0] != 0xFF) return false; + // Full 12-bit frame sync: top nibble of byte 1 is 0xF. + if ((c[1] & 0xF0) != 0xF0) return false; + // Layer bits (bits 2-1): 00=reserved, 01=III, 10=II, 11=I. + var layer = (c[1] >> 1) & 0x03; + if (layer == 0x00) return false; + return true; } }