From 8948ae873f21f26cb9d62010de82a120ce8cd121 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 13 Apr 2026 01:55:05 +0000 Subject: [PATCH 1/2] fix(security): expand MagicByteValidator beyond image-only allowlist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #601 added the audience-gated ChannelAttachmentPolicy that allows PDFs, Office documents, archives, and media for Team/Personal audiences — but MagicByteValidator's hardcoded AllowedExtensions dictionary still only accepted PNG/JPG/GIF/WebP, rejecting everything else at ingress with "File extension '.pdf' is not allowed". The SlackAttachmentIngressTests suite used NullContentScanner by default, so the Pdf_in_dm_* and Docx_in_dm_* tests only exercised the policy layer and never saw the real scanner. Rewrite MagicByteValidator around a signature-rule table keyed by declared MIME. Support every category the Team audience advertises: PDF, OOXML/ODF, legacy OLE Office, plain/structured text, RTF, zip/7z/rar/gzip/bzip2/xz, and mp3/mp4/wav/ogg/avi/webm/mkv. Harden each matcher beyond minimum magic — validate PDF version digit, ZIP exact header pair, gzip DEFLATE method, bzip2 BCD-Pi block header, RAR v4/v5 variant tail, ISO BMFF box size + printable-ASCII major brand, Ogg version byte, ID3v2 major version, MP3 strict 12-bit sync plus reserved-layer check. Seed ContentPolicy.DefaultAllowedMimeTypes from the validator's supported set so the two layers can't drift, and raise DefaultMaxFileSizeBytes from 20 MB to 25 MiB to match ChannelAttachmentPolicy. Flip SlackAttachmentIngressTests.BuildGateway to default to the real MagicByteContentScanner so the existing Pdf_in_dm_* and Docx_in_dm_* regression tests now actually exercise production behavior. Add a PlainText_in_dm_* ingress test, 13 MagicByteValidator category happy paths, and 15 adversarial polyglot-rejection tests for the hardened matchers. --- .../Channels/SlackAttachmentIngressTests.cs | 56 +- .../MagicByteValidatorTests.cs | 597 +++++++++++++++++- src/Netclaw.Security/ContentPolicy.cs | 27 +- src/Netclaw.Security/MagicByteValidator.cs | 497 +++++++++++---- 4 files changed, 1027 insertions(+), 150 deletions(-) diff --git a/src/Netclaw.Actors.Tests/Channels/SlackAttachmentIngressTests.cs b/src/Netclaw.Actors.Tests/Channels/SlackAttachmentIngressTests.cs index 997e5b395..40eae1b2e 100644 --- a/src/Netclaw.Actors.Tests/Channels/SlackAttachmentIngressTests.cs +++ b/src/Netclaw.Actors.Tests/Channels/SlackAttachmentIngressTests.cs @@ -40,6 +40,9 @@ public sealed class SlackAttachmentIngressVisionTests : TestKit private static readonly byte[] FakeDocxBytes = "PK\u0003\u0004fake docx content"u8.ToArray(); + private static readonly byte[] FakePlainTextBytes = + "meeting notes\n- discuss Q2 roadmap\n- assign OKRs\n"u8.ToArray(); + private readonly RecordingChatClient _chatClient = new(); private readonly RecordingReplyClient _replyClient = new(); private readonly ConfigurableFakeSlackFileHandler _httpHandler = new(); @@ -125,7 +128,7 @@ private IActorRef BuildGateway( BotUserId: new SlackUserId("UBOT"), DefaultChannelId: null, ReplyClient: _replyClient, - ContentScanner: scanner ?? new NullContentScanner(), + ContentScanner: scanner ?? new MagicByteContentScanner(new ContentPolicy()), ThreadHistoryFetcher: EmptyThreadHistoryFetcher.Instance, AudienceProfiles: profiles, ModelCapabilities: Host.Services.GetRequiredService(), @@ -437,6 +440,57 @@ await AwaitAssertAsync(() => }, duration: TimeSpan.FromSeconds(10), cancellationToken: TestContext.Current.CancellationToken); } + [Fact] + public async Task PlainText_in_dm_flows_through_real_magic_byte_scanner() + { + // Regression: before the MagicByteValidator rewrite, only image MIMEs + // passed the scanner — text/plain was rejected with + // "File extension '.txt' is not allowed" even though the policy layer + // allowed it. This test uses the default (real) MagicByteContentScanner + // to prove the broader category support end-to-end. + _httpHandler.RespondWith("text/plain", FakePlainTextBytes); + var gateway = BuildGateway("slack-gw-plaintext-flow"); + + var files = new List + { + new("F_TXT", "notes.txt", "text/plain", FakePlainTextBytes.Length, + "https://files.slack.com/files-pri/T1234-F_TXT/notes.txt") + }; + + gateway.Tell(new SlackInboundMessage( + Kind: SlackInboundKind.Message, + EventId: new SlackEventId("D_TXT:3800"), + ChannelId: new SlackChannelId("D_TXT"), + ThreadTs: null, + EventTs: new SlackEventTs("3800.1"), + UserId: new SlackUserId("U_HUMAN"), + BotId: null, + Text: "here are my notes", + Subtype: null, + Hidden: false, + IsDirectMessage: true, + Files: files)); + + await AwaitAssertAsync(() => + { + Assert.Contains(_chatClient.ReceivedMessages, + contents => contents.Any(c => c is TextContent t + && t.Text.Contains("[attachment]", StringComparison.Ordinal) + && t.Text.Contains("notes.txt", StringComparison.Ordinal) + && t.Text.Contains("path=\"inbox/notes.txt\"", StringComparison.Ordinal))); + }, duration: TimeSpan.FromSeconds(10), cancellationToken: TestContext.Current.CancellationToken); + + var sessionId = new SessionId("D_TXT/3800.1"); + var inboxPath = Path.Combine( + SessionDirectoryHelper.GetOrCreateInboxDirectory(sessionId, _paths.SessionsDirectory), + "notes.txt"); + Assert.True(File.Exists(inboxPath), $"Expected inbox file at {inboxPath}"); + + // Scanner should not have posted any rejection reply. + Assert.DoesNotContain(_replyClient.PostedMessages, + m => m.Text.Contains("Content scanner rejected", StringComparison.Ordinal)); + } + [Fact] public async Task Scanner_rejection_surfaces_user_visible_reply_with_no_inbox_write() { diff --git a/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs b/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs index a371a6013..572ae3e60 100644 --- a/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs +++ b/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs @@ -1,30 +1,69 @@ +using System.Collections.Frozen; using Xunit; namespace Netclaw.Security.Tests; public sealed class MagicByteValidatorTests { - // Real PNG header bytes + // ── Image signatures ────────────────────────────────────────────────── private static readonly byte[] PngHeader = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00]; - - // Real JPEG header bytes private static readonly byte[] JpegHeader = [0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10]; - - // Real GIF header bytes (GIF89a) private static readonly byte[] GifHeader = [0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x01, 0x00, 0x01, 0x00]; - - // Real WebP header bytes (RIFF....WEBP) private static readonly byte[] WebpHeader = [0x52, 0x49, 0x46, 0x46, 0x24, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50, 0x56]; - // Windows EXE (MZ header) + // ── Document / PDF signatures ───────────────────────────────────────── + // %PDF-1.4 followed by a tiny catalog so the header has real-looking bytes + private static readonly byte[] PdfHeader = + "%PDF-1.4\n1 0 obj\n<< /Type /Catalog >>\nendobj\n"u8.ToArray(); + + // ZIP local file header (PK\x03\x04) — prefix of OOXML docx/xlsx/pptx and ZIP archives + private static readonly byte[] ZipHeader = + [0x50, 0x4B, 0x03, 0x04, 0x14, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00]; + + // OLE Compound Document — legacy .doc/.xls/.ppt + private static readonly byte[] OleHeader = + [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1, 0x00, 0x00]; + + // RTF header: {\rtf1\ansi... + private static readonly byte[] RtfHeader = "{\\rtf1\\ansi\\deff0"u8.ToArray(); + + private static readonly byte[] PlainTextBytes = "Hello, this is a plain text file with no magic signature.\n"u8.ToArray(); + + // ── Archive signatures ──────────────────────────────────────────────── + private static readonly byte[] SevenZipHeader = [0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C, 0x00, 0x04]; + private static readonly byte[] RarHeader = [0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00]; + private static readonly byte[] GzipHeader = [0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00]; + // bzip2: "BZh" + block size '9' + 6-byte BCD-Pi compressed block header (31 41 59 26 53 59) + private static readonly byte[] Bzip2Header = + [0x42, 0x5A, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59]; + private static readonly byte[] XzHeader = [0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00, 0x00, 0x04]; + + // ── Media signatures ────────────────────────────────────────────────── + // ISO BMFF: 4 bytes box size, "ftyp", brand, minor version, compatible brands + private static readonly byte[] Mp4FtypHeader = + [0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x6D, 0x70, 0x34, 0x32, 0x00, 0x00, 0x00, 0x00]; + // RIFF....WAVE + private static readonly byte[] WavHeader = + [0x52, 0x49, 0x46, 0x46, 0x24, 0x00, 0x00, 0x00, 0x57, 0x41, 0x56, 0x45, 0x66, 0x6D, 0x74, 0x20]; + // RIFF....AVI (trailing space) + private static readonly byte[] AviHeader = + [0x52, 0x49, 0x46, 0x46, 0x24, 0x00, 0x00, 0x00, 0x41, 0x56, 0x49, 0x20, 0x4C, 0x49, 0x53, 0x54]; + // ID3v2 tag prefix (used by most MP3s) + private static readonly byte[] Mp3Id3Header = [0x49, 0x44, 0x33, 0x03, 0x00, 0x00, 0x00, 0x00]; + // MP3 raw frame sync: 0xFF 0xFB MPEG-1 Layer 3 + private static readonly byte[] Mp3FrameHeader = [0xFF, 0xFB, 0x90, 0x00]; + // Ogg container + private static readonly byte[] OggHeader = [0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00]; + // Matroska / WebM EBML header + private static readonly byte[] EbmlHeader = [0x1A, 0x45, 0xDF, 0xA3, 0x9F, 0x42, 0x86, 0x81]; + + // ── Executable signatures ───────────────────────────────────────────── private static readonly byte[] ExeHeader = [0x4D, 0x5A, 0x90, 0x00, 0x03, 0x00, 0x00, 0x00]; - - // Linux ELF header private static readonly byte[] ElfHeader = [0x7F, 0x45, 0x4C, 0x46, 0x02, 0x01, 0x01, 0x00]; - - // Shebang script private static readonly byte[] ShebangHeader = [0x23, 0x21, 0x2F, 0x62, 0x69, 0x6E, 0x2F, 0x73, 0x68]; // #!/bin/sh + // ── Images ──────────────────────────────────────────────────────────── + [Fact] public void Validate_PngWithValidBytes_Allowed() { @@ -70,6 +109,319 @@ public void Validate_WebpWithValidBytes_Allowed() Assert.Null(result.Error); } + // ── PDF ─────────────────────────────────────────────────────────────── + + [Fact] + public void Validate_PdfWithValidMagicBytes_Allowed() + { + var result = MagicByteValidator.Validate(PdfHeader, "application/pdf", "report.pdf"); + + Assert.True(result.IsAllowed); + Assert.Null(result.Error); + } + + [Fact] + public void Validate_PdfExtensionWithPngPayload_MimeTypeMismatch() + { + var result = MagicByteValidator.Validate(PngHeader, "application/pdf", "fake.pdf"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_PdfPayloadDeclaredAsPng_MimeTypeMismatch() + { + var result = MagicByteValidator.Validate(PdfHeader, "image/png", "photo.png"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + // ── OOXML / OLE / ODF documents ─────────────────────────────────────── + + [Fact] + public void Validate_DocxWithZipMagic_Allowed() + { + var result = MagicByteValidator.Validate( + ZipHeader, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "report.docx"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_XlsxWithZipMagic_Allowed() + { + var result = MagicByteValidator.Validate( + ZipHeader, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "budget.xlsx"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_PptxWithZipMagic_Allowed() + { + var result = MagicByteValidator.Validate( + ZipHeader, + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "slides.pptx"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_OdtWithZipMagic_Allowed() + { + var result = MagicByteValidator.Validate( + ZipHeader, + "application/vnd.oasis.opendocument.text", + "notes.odt"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_DocxWithOleHeader_MimeTypeMismatch() + { + // Old .doc bytes declared as .docx (OOXML) — mismatch + var result = MagicByteValidator.Validate( + OleHeader, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "report.docx"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_LegacyDocWithOleHeader_Allowed() + { + var result = MagicByteValidator.Validate(OleHeader, "application/msword", "legacy.doc"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_LegacyXlsWithOleHeader_Allowed() + { + var result = MagicByteValidator.Validate(OleHeader, "application/vnd.ms-excel", "legacy.xls"); + + Assert.True(result.IsAllowed); + } + + // ── Plain / structured text ─────────────────────────────────────────── + + [Fact] + public void Validate_PlainTextWithoutMagic_Allowed() + { + var result = MagicByteValidator.Validate(PlainTextBytes, "text/plain", "notes.txt"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_MarkdownAllowed() + { + var result = MagicByteValidator.Validate(PlainTextBytes, "text/markdown", "readme.md"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_CsvAllowed() + { + var result = MagicByteValidator.Validate(PlainTextBytes, "text/csv", "data.csv"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_JsonAllowed() + { + var json = "{\"ok\":true}\n"u8.ToArray(); + var result = MagicByteValidator.Validate(json, "application/json", "payload.json"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_PlainTextWithExecutablePrefix_RejectedAsExecutable() + { + // Someone renames a Windows EXE to .txt — the executable pre-check + // fires regardless of declared MIME, protecting the text/plain path. + var result = MagicByteValidator.Validate(ExeHeader, "text/plain", "notes.txt"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.ExecutableContent, result.Error); + } + + [Fact] + public void Validate_RtfAllowed() + { + var result = MagicByteValidator.Validate(RtfHeader, "application/rtf", "document.rtf"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_RtfWithBogusMagic_MimeTypeMismatch() + { + var result = MagicByteValidator.Validate(PngHeader, "application/rtf", "fake.rtf"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + // ── Archives ────────────────────────────────────────────────────────── + + [Fact] + public void Validate_ZipAllowed() + { + var result = MagicByteValidator.Validate(ZipHeader, "application/zip", "archive.zip"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_SevenZipAllowed() + { + var result = MagicByteValidator.Validate(SevenZipHeader, "application/x-7z-compressed", "archive.7z"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_RarAllowed() + { + var result = MagicByteValidator.Validate(RarHeader, "application/vnd.rar", "archive.rar"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_GzipAllowed() + { + var result = MagicByteValidator.Validate(GzipHeader, "application/gzip", "log.gz"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_Bzip2Allowed() + { + var result = MagicByteValidator.Validate(Bzip2Header, "application/x-bzip2", "backup.bz2"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_XzAllowed() + { + var result = MagicByteValidator.Validate(XzHeader, "application/x-xz", "backup.xz"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_ZipDeclaredAs7z_MimeTypeMismatch() + { + var result = MagicByteValidator.Validate(ZipHeader, "application/x-7z-compressed", "fake.7z"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + // ── Media ───────────────────────────────────────────────────────────── + + [Fact] + public void Validate_Mp4WithFtypBox_Allowed() + { + var result = MagicByteValidator.Validate(Mp4FtypHeader, "video/mp4", "clip.mp4"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_QuickTimeWithFtypBox_Allowed() + { + var result = MagicByteValidator.Validate(Mp4FtypHeader, "video/quicktime", "clip.mov"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_Mp3WithId3Tag_Allowed() + { + var result = MagicByteValidator.Validate(Mp3Id3Header, "audio/mpeg", "song.mp3"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_Mp3WithFrameSync_Allowed() + { + var result = MagicByteValidator.Validate(Mp3FrameHeader, "audio/mpeg", "song.mp3"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_WavAllowed() + { + var result = MagicByteValidator.Validate(WavHeader, "audio/wav", "sound.wav"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_AviAllowed() + { + var result = MagicByteValidator.Validate(AviHeader, "video/x-msvideo", "clip.avi"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_OggAllowed() + { + var result = MagicByteValidator.Validate(OggHeader, "audio/ogg", "sound.ogg"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_WebmAllowed() + { + var result = MagicByteValidator.Validate(EbmlHeader, "video/webm", "clip.webm"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_MatroskaAllowed() + { + var result = MagicByteValidator.Validate(EbmlHeader, "video/x-matroska", "clip.mkv"); + + Assert.True(result.IsAllowed); + } + + [Fact] + public void Validate_WavDeclaredAsMp4_MimeTypeMismatch() + { + // Both are "RIFF" but differ at offset 8 — WAV at offset 8 is "WAVE", + // MP4 at offset 4 is "ftyp". Make sure the stricter MP4 check rejects. + var result = MagicByteValidator.Validate(WavHeader, "video/mp4", "fake.mp4"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + // ── Cross-cutting ───────────────────────────────────────────────────── + [Fact] public void Validate_EmptyContent_Rejected() { @@ -108,11 +460,10 @@ public void Validate_ShebangScript_AlwaysRejected() } [Fact] - public void Validate_PdfMimeType_RejectedAsUnrecognized() + public void Validate_UnknownMimeType_Rejected() { - // PDF is not in image-only allowlist - byte[] pdfBytes = [0x25, 0x50, 0x44, 0x46, 0x2D, 0x31, 0x2E, 0x34]; // %PDF-1.4 - var result = MagicByteValidator.Validate(pdfBytes, "application/pdf", "doc.pdf"); + // application/octet-stream is classified as Other and not in the rule table + var result = MagicByteValidator.Validate(PngHeader, "application/octet-stream", "data.bin"); Assert.False(result.IsAllowed); Assert.Equal(ContentScanError.UnrecognizedFileType, result.Error); @@ -121,7 +472,8 @@ public void Validate_PdfMimeType_RejectedAsUnrecognized() [Fact] public void Validate_MimeTypeMismatch_ExtensionDoesNotMatchDeclaredType() { - // PNG bytes but declared as JPEG + // PNG bytes but declared as JPEG → mismatch (rule lookup finds jpeg, + // extension .png is not in jpeg's {.jpg,.jpeg} set) var result = MagicByteValidator.Validate(PngHeader, "image/jpeg", "photo.png"); Assert.False(result.IsAllowed); @@ -139,10 +491,30 @@ public void Validate_MagicByteMismatch_JpegBytesWithPngDeclaration() } [Fact] - public void Validate_UnknownExtension_Rejected() + public void Validate_UnknownExtensionForKnownMime_MimeTypeMismatch() { + // PNG bytes with MIME image/png, but filename is .bmp — extension + // is not in image/png's extension set, so the rule rejects it + // as a mismatch (not UnrecognizedFileType — the MIME is known). var result = MagicByteValidator.Validate(PngHeader, "image/png", "photo.bmp"); + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_PolicyDisallowsKnownMime_Rejected() + { + // Operator restricts policy to PNG only; PDF is known to the validator + // but disallowed by the runtime policy layer. + var policy = new ContentPolicy + { + AllowedMimeTypes = new HashSet(StringComparer.OrdinalIgnoreCase) + { "image/png" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase) + }; + + var result = MagicByteValidator.Validate(PdfHeader, "application/pdf", "report.pdf", policy); + Assert.False(result.IsAllowed); Assert.Equal(ContentScanError.UnrecognizedFileType, result.Error); } @@ -165,6 +537,7 @@ public void HasExecutableSignature_DetectsAllKnownSignatures() Assert.True(MagicByteValidator.HasExecutableSignature(ShebangHeader)); Assert.False(MagicByteValidator.HasExecutableSignature(PngHeader)); Assert.False(MagicByteValidator.HasExecutableSignature(JpegHeader)); + Assert.False(MagicByteValidator.HasExecutableSignature(PdfHeader)); } [Theory] @@ -172,7 +545,16 @@ public void HasExecutableSignature_DetectsAllKnownSignatures() [InlineData(nameof(JpegHeader), "image/jpeg")] [InlineData(nameof(GifHeader), "image/gif")] [InlineData(nameof(WebpHeader), "image/webp")] - public void DetectMimeType_identifies_supported_image_types(string headerField, string expectedMime) + [InlineData(nameof(PdfHeader), "application/pdf")] + [InlineData(nameof(ZipHeader), "application/zip")] + [InlineData(nameof(OleHeader), "application/x-ole-compound-document")] + [InlineData(nameof(Mp4FtypHeader), "video/mp4")] + [InlineData(nameof(WavHeader), "audio/wav")] + [InlineData(nameof(AviHeader), "video/x-msvideo")] + [InlineData(nameof(EbmlHeader), "video/webm")] + [InlineData(nameof(OggHeader), "audio/ogg")] + [InlineData(nameof(Mp3Id3Header), "audio/mpeg")] + public void DetectMimeType_identifies_supported_signature_families(string headerField, string expectedMime) { var header = headerField switch { @@ -180,12 +562,189 @@ public void DetectMimeType_identifies_supported_image_types(string headerField, nameof(JpegHeader) => JpegHeader, nameof(GifHeader) => GifHeader, nameof(WebpHeader) => WebpHeader, + nameof(PdfHeader) => PdfHeader, + nameof(ZipHeader) => ZipHeader, + nameof(OleHeader) => OleHeader, + nameof(Mp4FtypHeader) => Mp4FtypHeader, + nameof(WavHeader) => WavHeader, + nameof(AviHeader) => AviHeader, + nameof(EbmlHeader) => EbmlHeader, + nameof(OggHeader) => OggHeader, + nameof(Mp3Id3Header) => Mp3Id3Header, _ => throw new ArgumentException(headerField) }; Assert.Equal(expectedMime, MagicByteValidator.DetectMimeType(header)); } + // ── Hardening: reject minimum-magic polyglots ───────────────────────── + + [Fact] + public void Validate_RejectsJpegWithStuffedMarker() + { + // FF D8 FF 00 — FF 00 is an escaped stuffed byte inside JPEG data, + // not a valid JPEG start-of-image marker. A minimum-magic checker + // would accept this as JPEG; the hardened check rejects it. + byte[] bogusJpeg = [0xFF, 0xD8, 0xFF, 0x00, 0x10, 0x4A]; + var result = MagicByteValidator.Validate(bogusJpeg, "image/jpeg", "bogus.jpg"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsGifWithBogusVersion() + { + // GIF8Xa — minimum check (GIF8 prefix) would accept; tightened check + // requires '7a' or '9a' at bytes 4-5. + byte[] bogusGif = [0x47, 0x49, 0x46, 0x38, 0x58, 0x61, 0x01]; + var result = MagicByteValidator.Validate(bogusGif, "image/gif", "bogus.gif"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsPdfWithoutVersionDigit() + { + // %PDF-X.0 — missing digit after the dash. Minimum 5-byte check + // would accept; tightened check requires a digit + dot. + byte[] bogusPdf = "%PDF-X.0\n"u8.ToArray(); + var result = MagicByteValidator.Validate(bogusPdf, "application/pdf", "bogus.pdf"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsRtfWithoutVersionDigit() + { + // {\rtfX — RTF spec requires a version digit after \rtf. + byte[] bogusRtf = "{\\rtfX junk"u8.ToArray(); + var result = MagicByteValidator.Validate(bogusRtf, "application/rtf", "bogus.rtf"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsZipWithInvalidHeaderPair() + { + // PK\x03\x06 is not a valid ZIP local/central-dir/spanned marker. + // Minimum check (PK + any of 03/05/07 + any of 04/06/08) would have + // accepted this; tightened check requires the exact 4-byte pair. + byte[] bogusZip = [0x50, 0x4B, 0x03, 0x06, 0x14, 0x00, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusZip, "application/zip", "bogus.zip"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsGzipWithNonDeflateMethod() + { + // 1F 8B 01 — compression method 1 is reserved; only 0x08 (DEFLATE) + // is defined by RFC 1952. Minimum 2-byte check would accept. + byte[] bogusGzip = [0x1F, 0x8B, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusGzip, "application/gzip", "bogus.gz"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsBzip2WithBogusBlockHeader() + { + // BZh9 + garbage where the BCD-Pi block header should be. + byte[] bogusBzip2 = [0x42, 0x5A, 0x68, 0x39, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]; + var result = MagicByteValidator.Validate(bogusBzip2, "application/x-bzip2", "bogus.bz2"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsFtypWithNonAsciiMajorBrand() + { + // Valid "ftyp" box header but major brand bytes are not printable + // ASCII — indicates a garbage or adversarial box payload. + byte[] bogusFtyp = + [0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x00, 0x01, 0x02, 0x03]; + var result = MagicByteValidator.Validate(bogusFtyp, "video/mp4", "bogus.mp4"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsOggWithNonZeroVersion() + { + // OggS + version byte 0x01 — RFC 3533 requires version 0x00. + byte[] bogusOgg = [0x4F, 0x67, 0x67, 0x53, 0x01, 0x02, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusOgg, "audio/ogg", "bogus.ogg"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsMp3WithReservedLayer() + { + // 0xFF 0xF9 — top nibble F (12-bit sync OK), version bits 11 + // (MPEG-1 OK), but layer bits (2-1) = 00 → reserved per ISO/IEC + // 11172-3. 0xF9 = 1111 1001, bits 2-1 = 00. + byte[] bogusMp3 = [0xFF, 0xF9, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusMp3, "audio/mpeg", "bogus.mp3"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsMp3With11BitSyncPolyglot() + { + // 0xFF 0xE0 — only 11-bit sync. The looser check accepted this; + // the tightened 12-bit sync requires top nibble F, which rejects. + byte[] bogusMp3 = [0xFF, 0xE0, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusMp3, "audio/mpeg", "bogus.mp3"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsId3WithBogusMajorVersion() + { + // ID3 + major version 99 — valid ID3v2 is 2, 3, or 4. + byte[] bogusId3 = [0x49, 0x44, 0x33, 0x63, 0x00, 0x00, 0x00, 0x00]; + var result = MagicByteValidator.Validate(bogusId3, "audio/mpeg", "bogus.mp3"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_RejectsRarWithoutValidVariantTail() + { + // Rar!\x1A\x07\xFF — matches the looser 6-byte prefix, but the + // tightened check requires v4 (0x00) or v5 (0x01 0x00). + byte[] bogusRar = [0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0xFF]; + var result = MagicByteValidator.Validate(bogusRar, "application/vnd.rar", "bogus.rar"); + + Assert.False(result.IsAllowed); + Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); + } + + [Fact] + public void Validate_AcceptsRarV5WithEightByteMagic() + { + // RAR v5 signature: 8 bytes ending in 01 00. + byte[] rarV5 = [0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00, 0x00, 0x00]; + var result = MagicByteValidator.Validate(rarV5, "application/vnd.rar", "archive.rar"); + + Assert.True(result.IsAllowed); + } + [Fact] public void DetectMimeType_returns_null_for_unknown_content() { diff --git a/src/Netclaw.Security/ContentPolicy.cs b/src/Netclaw.Security/ContentPolicy.cs index 92b0fda73..348e72638 100644 --- a/src/Netclaw.Security/ContentPolicy.cs +++ b/src/Netclaw.Security/ContentPolicy.cs @@ -3,18 +3,26 @@ namespace Netclaw.Security; /// -/// Configurable content security policy for file uploads. +/// Configurable content security policy for file uploads. Layered on top of +/// : the validator advertises which MIME +/// types it knows how to verify, and this policy is the runtime allowlist +/// that operators can use to further restrict what passes. /// public sealed class ContentPolicy { /// - /// Default maximum file size: 20 MB. + /// Default maximum file size: 25 MiB. Tracks + /// ChannelAttachmentPolicy.DefaultMaxFileBytes so the scanner + /// ceiling and the channel policy ceiling don't drift. /// - public const long DefaultMaxFileSizeBytes = 20 * 1024 * 1024; + public const long DefaultMaxFileSizeBytes = 25L * 1024 * 1024; /// - /// MIME types allowed through the content scanner. - /// Defaults to image types supported by vision models. + /// MIME types allowed through the content scanner. Defaults to the full + /// set of MIME types knows how to verify + /// — images, PDF, Office documents, plain/rich text, archives, and + /// common audio/video containers. Operators can override this to + /// restrict the allowlist further. /// public FrozenSet AllowedMimeTypes { get; init; } = DefaultAllowedMimeTypes; @@ -24,11 +32,6 @@ public sealed class ContentPolicy public long MaxFileSizeBytes { get; init; } = DefaultMaxFileSizeBytes; public static readonly FrozenSet DefaultAllowedMimeTypes = - new HashSet(StringComparer.OrdinalIgnoreCase) - { - "image/png", - "image/jpeg", - "image/gif", - "image/webp" - }.ToFrozenSet(StringComparer.OrdinalIgnoreCase); + MagicByteValidator.GetSupportedMimeTypes() + .ToFrozenSet(StringComparer.OrdinalIgnoreCase); } diff --git a/src/Netclaw.Security/MagicByteValidator.cs b/src/Netclaw.Security/MagicByteValidator.cs index 0431b7d62..802f59fbd 100644 --- a/src/Netclaw.Security/MagicByteValidator.cs +++ b/src/Netclaw.Security/MagicByteValidator.cs @@ -4,31 +4,17 @@ namespace Netclaw.Security; /// /// Validates file content using magic byte (file signature) analysis. -/// Narrowed to image types for Netclaw's multimodal pipeline. +/// Supports the categories advertised by ChannelAttachmentPolicy's +/// Team / Personal audience defaults: images, PDFs, Office +/// documents (OOXML + legacy OLE), plain/rich text, common archives, and +/// common audio/video containers. Unknown declared MIME types are rejected +/// as . /// public static class MagicByteValidator { - private static readonly FrozenSet AllowedImageMimeTypes = - new HashSet(StringComparer.OrdinalIgnoreCase) - { - "image/png", - "image/jpeg", - "image/gif", - "image/webp" - }.ToFrozenSet(StringComparer.OrdinalIgnoreCase); - - private static readonly FrozenDictionary> AllowedExtensions = - new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - [".png"] = new[] { "image/png" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase), - [".jpg"] = new[] { "image/jpeg" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase), - [".jpeg"] = new[] { "image/jpeg" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase), - [".gif"] = new[] { "image/gif" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase), - [".webp"] = new[] { "image/webp" }.ToFrozenSet(StringComparer.OrdinalIgnoreCase), - }.ToFrozenDictionary(StringComparer.OrdinalIgnoreCase); - /// - /// Magic byte signatures for executable content — always blocked. + /// Magic byte signatures for executable content — always blocked, + /// regardless of declared MIME type. /// private static readonly byte[][] ExecutableSignatures = [ @@ -40,9 +26,39 @@ public static class MagicByteValidator [0x23, 0x21] // #! — Shebang (shell scripts) ]; + /// + /// Matcher delegate for a signature rule. Uses a typed delegate instead + /// of Func<ReadOnlySpan<byte>, bool> because + /// cannot be a generic type argument. + /// + private delegate bool SignatureMatcher(ReadOnlySpan content); + + /// + /// Signature rule for a declared MIME type. Extensions are the set of + /// filename extensions permitted to carry this MIME; + /// returns true when the raw bytes satisfy the signature family. + /// + private sealed record SignatureRule( + FrozenSet Extensions, + SignatureMatcher Matches); + + /// + /// Matcher that unconditionally accepts any content. Used for text-like + /// MIMEs that have no signature at offset 0; the executable pre-check + /// still runs first, so an MZ- or #!-prefixed "text" file + /// is still rejected as . + /// + private static readonly SignatureMatcher AnyContent = static _ => true; + + private static readonly FrozenDictionary RulesByMime = + BuildRules().ToFrozenDictionary(StringComparer.OrdinalIgnoreCase); + /// /// Validates file content against its declared MIME type and filename. - /// Only image types are allowed through. + /// Rejects empty content, oversized content, executables, unknown or + /// disallowed MIME types, filenames whose extension doesn't match the + /// declared MIME type, and content whose magic bytes don't match the + /// declared signature family. /// public static ContentScanResult Validate( ReadOnlySpan content, @@ -58,16 +74,14 @@ public static ContentScanResult Validate( } var effectivePolicy = policy ?? new ContentPolicy(); - var allowedMimes = effectivePolicy.AllowedMimeTypes; if (content.Length > effectivePolicy.MaxFileSizeBytes) { return ContentScanResult.Rejected( ContentScanError.FileTooLarge, - $"File exceeds maximum size of {effectivePolicy.MaxFileSizeBytes / (1024 * 1024)} MB"); + $"File exceeds maximum size of {effectivePolicy.MaxFileSizeBytes / (1024 * 1024)} MiB"); } - // Always check for executables — these are never allowed if (HasExecutableSignature(content)) { var detectedType = DetectMimeType(content); @@ -77,65 +91,74 @@ public static ContentScanResult Validate( detectedType is not null ? new MimeType(detectedType) : null); } - // Validate extension is in allowlist - var extension = Path.GetExtension(filename); - if (string.IsNullOrEmpty(extension) || !AllowedExtensions.ContainsKey(extension)) + if (!RulesByMime.TryGetValue(declaredMimeType, out var rule)) { return ContentScanResult.Rejected( ContentScanError.UnrecognizedFileType, - $"File extension '{extension}' is not allowed"); + $"MIME type '{declaredMimeType}' is not supported by the content scanner"); } - // Validate declared MIME type is allowed - if (!allowedMimes.Contains(declaredMimeType)) + if (!effectivePolicy.AllowedMimeTypes.Contains(declaredMimeType)) { return ContentScanResult.Rejected( ContentScanError.UnrecognizedFileType, - $"MIME type '{declaredMimeType}' is not allowed"); + $"MIME type '{declaredMimeType}' is not allowed by policy"); } - // Validate extension matches declared MIME type - var allowedMimesForExtension = AllowedExtensions[extension]; - if (!allowedMimesForExtension.Contains(declaredMimeType)) + var extension = Path.GetExtension(filename); + if (string.IsNullOrEmpty(extension)) + { + return ContentScanResult.Rejected( + ContentScanError.UnrecognizedFileType, + "File has no extension"); + } + + if (!rule.Extensions.Contains(extension)) { return ContentScanResult.Rejected( ContentScanError.MimeTypeMismatch, $"Extension '{extension}' does not match declared type '{declaredMimeType}'"); } - // Validate magic bytes match expected image type - return ValidateImageContent(content, declaredMimeType); + if (!rule.Matches(content)) + { + var detectedMimeType = DetectMimeType(content); + return ContentScanResult.Rejected( + ContentScanError.MimeTypeMismatch, + $"Content is not a valid {declaredMimeType} file", + detectedMimeType is not null ? new MimeType(detectedMimeType) : null); + } + + return ContentScanResult.Allowed(new MimeType(declaredMimeType)); } /// - /// Detects MIME type from magic bytes for the supported image types. + /// Detects MIME type from magic bytes for supported signature families. + /// Used to populate the DetectedType field on rejection results so + /// operators can see "looks like a PDF but was declared as an image". /// Returns null for unrecognized content. /// public static string? DetectMimeType(ReadOnlySpan content) { - if (content.Length >= 8 && - content[0] == 0x89 && content[1] == 0x50 && - content[2] == 0x4E && content[3] == 0x47 && - content[4] == 0x0D && content[5] == 0x0A && - content[6] == 0x1A && content[7] == 0x0A) - return "image/png"; - - if (content.Length >= 3 && - content[0] == 0xFF && content[1] == 0xD8 && content[2] == 0xFF) - return "image/jpeg"; - - if (content.Length >= 4 && - content[0] == 0x47 && content[1] == 0x49 && - content[2] == 0x46 && content[3] == 0x38) - return "image/gif"; - - if (content.Length >= 12 && - content[0] == 0x52 && content[1] == 0x49 && - content[2] == 0x46 && content[3] == 0x46 && - content[8] == 0x57 && content[9] == 0x45 && - content[10] == 0x42 && content[11] == 0x50) - return "image/webp"; - + if (IsPng(content)) return "image/png"; + if (IsJpeg(content)) return "image/jpeg"; + if (IsGif(content)) return "image/gif"; + if (IsWebp(content)) return "image/webp"; + if (IsPdf(content)) return "application/pdf"; + if (IsOle(content)) return "application/x-ole-compound-document"; + if (IsRtf(content)) return "application/rtf"; + if (Is7z(content)) return "application/x-7z-compressed"; + if (IsRar(content)) return "application/vnd.rar"; + if (IsGzip(content)) return "application/gzip"; + if (IsBzip2(content)) return "application/x-bzip2"; + if (IsXz(content)) return "application/x-xz"; + if (IsZip(content)) return "application/zip"; + if (IsWav(content)) return "audio/wav"; + if (IsAvi(content)) return "video/x-msvideo"; + if (IsFtyp(content)) return "video/mp4"; + if (IsEbml(content)) return "video/webm"; + if (IsOgg(content)) return "audio/ogg"; + if (IsMp3FrameOrId3(content)) return "audio/mpeg"; return null; } @@ -153,69 +176,307 @@ public static bool HasExecutableSignature(ReadOnlySpan content) return false; } - private static ContentScanResult ValidateImageContent( - ReadOnlySpan content, - string declaredMimeType) + // ── Rule table ──────────────────────────────────────────────────────── + + private static Dictionary BuildRules() { - var detectedMimeType = DetectMimeType(content); + var rules = new Dictionary(StringComparer.OrdinalIgnoreCase); - if (declaredMimeType.Equals("image/png", StringComparison.OrdinalIgnoreCase)) - { - // PNG: 89 50 4E 47 0D 0A 1A 0A - if (content.Length < 8 || - content[0] != 0x89 || content[1] != 0x50 || - content[2] != 0x4E || content[3] != 0x47 || - content[4] != 0x0D || content[5] != 0x0A || - content[6] != 0x1A || content[7] != 0x0A) - { - return ContentScanResult.Rejected( - ContentScanError.MimeTypeMismatch, - "Content is not a valid PNG file", - detectedMimeType is not null ? new MimeType(detectedMimeType) : null); - } - } - else if (declaredMimeType.Equals("image/jpeg", StringComparison.OrdinalIgnoreCase)) - { - // JPEG: FF D8 FF - if (content.Length < 3 || - content[0] != 0xFF || content[1] != 0xD8 || content[2] != 0xFF) - { - return ContentScanResult.Rejected( - ContentScanError.MimeTypeMismatch, - "Content is not a valid JPEG file", - detectedMimeType is not null ? new MimeType(detectedMimeType) : null); - } - } - else if (declaredMimeType.Equals("image/gif", StringComparison.OrdinalIgnoreCase)) + // Images + rules["image/png"] = new(Exts(".png"), IsPng); + rules["image/jpeg"] = new(Exts(".jpg", ".jpeg"), IsJpeg); + rules["image/gif"] = new(Exts(".gif"), IsGif); + rules["image/webp"] = new(Exts(".webp"), IsWebp); + + // PDF + rules["application/pdf"] = new(Exts(".pdf"), IsPdf); + + // OOXML (ZIP-based) — docx/xlsx/pptx + rules["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = + new(Exts(".docx"), IsZip); + rules["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = + new(Exts(".xlsx"), IsZip); + rules["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = + new(Exts(".pptx"), IsZip); + + // OpenDocument (also ZIP-based) + rules["application/vnd.oasis.opendocument.text"] = new(Exts(".odt"), IsZip); + rules["application/vnd.oasis.opendocument.spreadsheet"] = new(Exts(".ods"), IsZip); + rules["application/vnd.oasis.opendocument.presentation"] = new(Exts(".odp"), IsZip); + + // Legacy OLE Compound Document Office formats + rules["application/msword"] = new(Exts(".doc"), IsOle); + rules["application/vnd.ms-excel"] = new(Exts(".xls"), IsOle); + rules["application/vnd.ms-powerpoint"] = new(Exts(".ppt"), IsOle); + + // Plain/structured text — no signature, but executable pre-check + // still blocks MZ / ELF / shebang payloads. + rules["text/plain"] = new(Exts(".txt", ".log"), AnyContent); + rules["text/markdown"] = new(Exts(".md", ".markdown"), AnyContent); + rules["text/csv"] = new(Exts(".csv"), AnyContent); + rules["text/xml"] = new(Exts(".xml"), AnyContent); + rules["application/json"] = new(Exts(".json"), AnyContent); + rules["application/xml"] = new(Exts(".xml"), AnyContent); + rules["application/yaml"] = new(Exts(".yml", ".yaml"), AnyContent); + rules["application/x-yaml"] = new(Exts(".yml", ".yaml"), AnyContent); + + // Rich text + rules["application/rtf"] = new(Exts(".rtf"), IsRtf); + rules["text/rtf"] = new(Exts(".rtf"), IsRtf); + + // Archives + rules["application/zip"] = new(Exts(".zip"), IsZip); + rules["application/x-zip-compressed"] = new(Exts(".zip"), IsZip); + rules["application/x-7z-compressed"] = new(Exts(".7z"), Is7z); + rules["application/vnd.rar"] = new(Exts(".rar"), IsRar); + rules["application/x-rar-compressed"] = new(Exts(".rar"), IsRar); + rules["application/gzip"] = new(Exts(".gz", ".tgz"), IsGzip); + rules["application/x-gzip"] = new(Exts(".gz", ".tgz"), IsGzip); + rules["application/x-bzip2"] = new(Exts(".bz2"), IsBzip2); + rules["application/x-xz"] = new(Exts(".xz"), IsXz); + + // Audio + rules["audio/mpeg"] = new(Exts(".mp3"), IsMp3FrameOrId3); + rules["audio/mp4"] = new(Exts(".m4a", ".mp4"), IsFtyp); + rules["audio/x-m4a"] = new(Exts(".m4a"), IsFtyp); + rules["audio/wav"] = new(Exts(".wav"), IsWav); + rules["audio/x-wav"] = new(Exts(".wav"), IsWav); + rules["audio/ogg"] = new(Exts(".ogg", ".oga"), IsOgg); + + // Video + rules["video/mp4"] = new(Exts(".mp4", ".m4v"), IsFtyp); + rules["video/quicktime"] = new(Exts(".mov"), IsFtyp); + rules["video/webm"] = new(Exts(".webm"), IsEbml); + rules["video/x-matroska"] = new(Exts(".mkv"), IsEbml); + rules["video/x-msvideo"] = new(Exts(".avi"), IsAvi); + + return rules; + } + + private static FrozenSet Exts(params string[] extensions) => + extensions.ToFrozenSet(StringComparer.OrdinalIgnoreCase); + + /// + /// Internal helper exposed for to seed its + /// default allowlist from the set of MIME types the validator knows how + /// to verify, so the two layers stay in sync. + /// + internal static IEnumerable GetSupportedMimeTypes() => RulesByMime.Keys; + + // ── Signature matchers ──────────────────────────────────────────────── + // Each matcher validates more than the minimum prefix where a cheap + // follow-on check is available. The goal is to reject polyglots and + // hand-forged headers that match the minimum magic but have bogus + // version/layer/brand fields, without false-rejecting real files. + + private static bool IsPng(ReadOnlySpan c) => + c.Length >= 8 && + c[0] == 0x89 && c[1] == 0x50 && c[2] == 0x4E && c[3] == 0x47 && + c[4] == 0x0D && c[5] == 0x0A && c[6] == 0x1A && c[7] == 0x0A; + + /// + /// JPEG: SOI (FF D8) followed by a marker byte. The third byte + /// must look like a JPEG marker (high nibble F, low nibble not + /// 0 or F — FF 00 is a stuffed byte inside data, + /// FF FF is fill). + /// + private static bool IsJpeg(ReadOnlySpan c) + { + if (c.Length < 3) return false; + if (c[0] != 0xFF || c[1] != 0xD8 || c[2] != 0xFF) return false; + if (c.Length < 4) return false; + var marker = c[3]; + return marker != 0x00 && marker != 0xFF; + } + + /// + /// GIF: GIF87a or GIF89a — validate all 6 bytes rather + /// than just the GIF8 prefix so GIF8Xa prefixes from + /// adversarial content are rejected. + /// + private static bool IsGif(ReadOnlySpan c) + { + if (c.Length < 6) return false; + if (c[0] != 0x47 || c[1] != 0x49 || c[2] != 0x46 || c[3] != 0x38) return false; + if (c[5] != 0x61) return false; // 'a' + return c[4] == 0x37 || c[4] == 0x39; // '7' or '9' + } + + private static bool IsRiff(ReadOnlySpan c) => + c.Length >= 4 && + c[0] == 0x52 && c[1] == 0x49 && c[2] == 0x46 && c[3] == 0x46; + + private static bool IsWebp(ReadOnlySpan c) => + IsRiff(c) && c.Length >= 12 && + c[8] == 0x57 && c[9] == 0x45 && c[10] == 0x42 && c[11] == 0x50; + + private static bool IsWav(ReadOnlySpan c) => + IsRiff(c) && c.Length >= 12 && + c[8] == 0x57 && c[9] == 0x41 && c[10] == 0x56 && c[11] == 0x45; + + private static bool IsAvi(ReadOnlySpan c) => + IsRiff(c) && c.Length >= 12 && + c[8] == 0x41 && c[9] == 0x56 && c[10] == 0x49 && c[11] == 0x20; + + /// + /// PDF: %PDF- followed by a version digit and a dot. A minimal + /// valid PDF starts with %PDF-1.0 through %PDF-2.0; the + /// tightened check requires byte 5 to be an ASCII digit and byte 6 to + /// be a dot, rejecting %PDF-xx garbage headers. + /// + private static bool IsPdf(ReadOnlySpan c) + { + if (c.Length < 7) return false; + if (c[0] != 0x25 || c[1] != 0x50 || c[2] != 0x44 || c[3] != 0x46 || c[4] != 0x2D) + return false; + return c[5] is >= (byte)'0' and <= (byte)'9' && c[6] == (byte)'.'; + } + + /// + /// ZIP local file header (PK\x03\x04), central directory end + /// (PK\x05\x06), or spanned-archive marker (PK\x07\x08). + /// The third and fourth bytes must form one of those three exact pairs + /// to reject PK\x03\x06-style garbage. + /// + private static bool IsZip(ReadOnlySpan c) + { + if (c.Length < 4) return false; + if (c[0] != 0x50 || c[1] != 0x4B) return false; + return (c[2] == 0x03 && c[3] == 0x04) + || (c[2] == 0x05 && c[3] == 0x06) + || (c[2] == 0x07 && c[3] == 0x08); + } + + private static bool IsOle(ReadOnlySpan c) => + c.Length >= 8 && + c[0] == 0xD0 && c[1] == 0xCF && c[2] == 0x11 && c[3] == 0xE0 && + c[4] == 0xA1 && c[5] == 0xB1 && c[6] == 0x1A && c[7] == 0xE1; + + /// + /// RTF: {\rtf followed by a version digit (RTF spec requires + /// {\rtf1 — the 1 in practice, but the spec allows any + /// digit for the version field). + /// + private static bool IsRtf(ReadOnlySpan c) + { + if (c.Length < 6) return false; + if (c[0] != 0x7B || c[1] != 0x5C || c[2] != 0x72 || c[3] != 0x74 || c[4] != 0x66) + return false; + return c[5] is >= (byte)'0' and <= (byte)'9'; + } + + private static bool Is7z(ReadOnlySpan c) => + c.Length >= 6 && + c[0] == 0x37 && c[1] == 0x7A && c[2] == 0xBC && c[3] == 0xAF && + c[4] == 0x27 && c[5] == 0x1C; + + /// + /// RAR v1.5–v4 (Rar!\x1A\x07\x00, 7 bytes) or RAR v5+ + /// (Rar!\x1A\x07\x01\x00, 8 bytes). Tightened from the looser + /// 6-byte prefix to require the exact trailing bytes of one variant. + /// + private static bool IsRar(ReadOnlySpan c) + { + if (c.Length < 7) return false; + if (c[0] != 0x52 || c[1] != 0x61 || c[2] != 0x72 || c[3] != 0x21) return false; + if (c[4] != 0x1A || c[5] != 0x07) return false; + if (c[6] == 0x00) return true; // v4 + if (c[6] == 0x01 && c.Length >= 8 && c[7] == 0x00) return true; // v5 + return false; + } + + /// + /// Gzip: 1F 8B plus compression method 08 (DEFLATE). + /// RFC 1952 reserves methods 0–7; only 8 is defined. Rejecting + /// unknown methods blocks crafted gzip polyglots. + /// + private static bool IsGzip(ReadOnlySpan c) => + c.Length >= 3 && c[0] == 0x1F && c[1] == 0x8B && c[2] == 0x08; + + /// + /// Bzip2: BZh + block size digit (1–9, hundreds of + /// kilobytes) + the 6-byte BCD-encoded magic number 31 41 59 26 53 59 + /// (the first 6 digits of Pi — the bzip2 compressed-block header). + /// Validates full 10-byte prefix, not just BZh. + /// + private static bool IsBzip2(ReadOnlySpan c) + { + if (c.Length < 10) return false; + if (c[0] != 0x42 || c[1] != 0x5A || c[2] != 0x68) return false; + if (c[3] is < (byte)'1' or > (byte)'9') return false; + return c[4] == 0x31 && c[5] == 0x41 && c[6] == 0x59 + && c[7] == 0x26 && c[8] == 0x53 && c[9] == 0x59; + } + + private static bool IsXz(ReadOnlySpan c) => + c.Length >= 6 && + c[0] == 0xFD && c[1] == 0x37 && c[2] == 0x7A && c[3] == 0x58 && + c[4] == 0x5A && c[5] == 0x00; + + /// + /// ISO BMFF: 4-byte box size (≥ 8), ftyp at offset 4, then a + /// 4-byte major brand which must be printable ASCII. Rejects headers + /// where the major brand is garbage — a common polyglot failure mode. + /// + private static bool IsFtyp(ReadOnlySpan c) + { + if (c.Length < 12) return false; + if (c[4] != 0x66 || c[5] != 0x74 || c[6] != 0x79 || c[7] != 0x70) return false; + + // Box size big-endian uint32 at bytes 0-3. Must be ≥ 8 and either + // representable (≤ content length + some slack) or the special + // value 0 (box extends to end-of-file) or 1 (64-bit size follows). + uint boxSize = ((uint)c[0] << 24) | ((uint)c[1] << 16) | ((uint)c[2] << 8) | c[3]; + if (boxSize != 0 && boxSize != 1 && boxSize < 8) return false; + + // Major brand at bytes 8-11 must be printable ASCII. + for (var i = 8; i < 12; i++) { - // GIF: GIF8 (47 49 46 38) - if (content.Length < 4 || - content[0] != 0x47 || content[1] != 0x49 || - content[2] != 0x46 || content[3] != 0x38) - { - return ContentScanResult.Rejected( - ContentScanError.MimeTypeMismatch, - "Content is not a valid GIF file", - detectedMimeType is not null ? new MimeType(detectedMimeType) : null); - } + var b = c[i]; + if (b < 0x20 || b > 0x7E) return false; } - else if (declaredMimeType.Equals("image/webp", StringComparison.OrdinalIgnoreCase)) + return true; + } + + private static bool IsEbml(ReadOnlySpan c) => + c.Length >= 4 && + c[0] == 0x1A && c[1] == 0x45 && c[2] == 0xDF && c[3] == 0xA3; + + /// + /// Ogg: OggS followed by a version byte that per RFC 3533 §6 + /// must be 0x00. Rejects Ogg-prefixed polyglots with bogus version. + /// + private static bool IsOgg(ReadOnlySpan c) => + c.Length >= 5 && + c[0] == 0x4F && c[1] == 0x67 && c[2] == 0x67 && c[3] == 0x53 && + c[4] == 0x00; + + /// + /// MP3: either an ID3v2 tag (ID3 + valid major version 2/3/4) + /// or a raw MPEG audio frame sync. Uses the strict 12-bit sync + /// (FF Fx) and validates the layer bits are not reserved. + /// 12-bit sync is stricter than the 11-bit form — it rejects + /// MPEG-2.5 (rare/non-standard) and FF E* polyglots — and + /// implicitly rules out reserved MPEG version 01, since that would + /// require bit 4 = 0 (top nibble E, not F). Layer bits + /// (bits 2–1 of byte 1) are still checked: 00 is reserved + /// per ISO/IEC 11172-3 and must be rejected. + /// + private static bool IsMp3FrameOrId3(ReadOnlySpan c) + { + if (c.Length >= 4 && c[0] == 0x49 && c[1] == 0x44 && c[2] == 0x33) { - // WebP: RIFF....WEBP (52 49 46 46 xx xx xx xx 57 45 42 50) - if (content.Length < 12 || - content[0] != 0x52 || content[1] != 0x49 || - content[2] != 0x46 || content[3] != 0x46 || - content[8] != 0x57 || content[9] != 0x45 || - content[10] != 0x42 || content[11] != 0x50) - { - return ContentScanResult.Rejected( - ContentScanError.MimeTypeMismatch, - "Content is not a valid WebP file", - detectedMimeType is not null ? new MimeType(detectedMimeType) : null); - } + // ID3v2 major version (byte 3) is 2, 3, or 4 in the wild. + return c[3] is 0x02 or 0x03 or 0x04; } - return ContentScanResult.Allowed( - new MimeType(detectedMimeType ?? declaredMimeType)); + if (c.Length < 2) return false; + if (c[0] != 0xFF) return false; + // Full 12-bit frame sync: top nibble of byte 1 is 0xF. + if ((c[1] & 0xF0) != 0xF0) return false; + // Layer bits (bits 2-1): 00=reserved, 01=III, 10=II, 11=I. + var layer = (c[1] >> 1) & 0x03; + if (layer == 0x00) return false; + return true; } } From 5d8f49e3a3dc9a10b7acef35c66fd08bdcee3993 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 13 Apr 2026 10:37:23 +0000 Subject: [PATCH 2/2] fix slack attachment policy bypasses and fail-open scanning --- .../Channels/SlackFileFlowIntegrationTests.cs | 12 ++- .../SlackThreadBackfillIntegrationTests.cs | 101 ++++++++++++++++++ .../SlackThreadBindingActor.cs | 90 ++++++++++------ .../ChannelAttachmentPolicyTests.cs | 2 +- .../ChannelAttachmentPolicy.cs | 3 - .../MagicByteValidatorTests.cs | 31 ------ src/Netclaw.Security/MagicByteValidator.cs | 18 ---- 7 files changed, 169 insertions(+), 88 deletions(-) diff --git a/src/Netclaw.Actors.Tests/Channels/SlackFileFlowIntegrationTests.cs b/src/Netclaw.Actors.Tests/Channels/SlackFileFlowIntegrationTests.cs index b49cb705c..a76699105 100644 --- a/src/Netclaw.Actors.Tests/Channels/SlackFileFlowIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Channels/SlackFileFlowIntegrationTests.cs @@ -1174,10 +1174,9 @@ await AwaitAssertAsync(() => } [Fact] - public async Task Scanner_failure_does_not_silently_drop_image() + public async Task Scanner_failure_rejects_attachment_and_does_not_inline() { - // When the content scanner itself is broken (e.g. TypeInitializationException), - // images should still flow through to the LLM rather than being silently dropped. + // Scanner failures must fail closed for inbound attachments. var pipeline = Host.Services.GetRequiredService(); var httpClient = new HttpClient(_httpHandler); @@ -1231,8 +1230,11 @@ await AwaitAssertAsync(() => "Expected at least one Slack reply to be posted"); }, duration: TimeSpan.FromSeconds(10), cancellationToken: TestContext.Current.CancellationToken); - Assert.True(_chatClient.ReceivedImageContent, - "Expected LLM to receive image even when scanner is broken"); + Assert.False(_chatClient.ReceivedImageContent, + "Expected LLM not to receive image when scanner fails"); + + Assert.Contains(_replyClient.PostedMessages, + m => m.Text.Contains("Couldn't scan `drawing.png`", StringComparison.Ordinal)); } /// diff --git a/src/Netclaw.Actors.Tests/Channels/SlackThreadBackfillIntegrationTests.cs b/src/Netclaw.Actors.Tests/Channels/SlackThreadBackfillIntegrationTests.cs index 288799792..d0a9acd66 100644 --- a/src/Netclaw.Actors.Tests/Channels/SlackThreadBackfillIntegrationTests.cs +++ b/src/Netclaw.Actors.Tests/Channels/SlackThreadBackfillIntegrationTests.cs @@ -447,6 +447,107 @@ await AwaitAssertAsync(() => }, duration: TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken); } + [Fact] + public async Task Backfill_document_in_public_channel_is_not_forwarded_as_data_content() + { + var pipeline = Host.Services.GetRequiredService(); + var httpClient = new HttpClient(_httpHandler); + + var fetcher = new SlackThreadHistoryFetcher( + (channelId, threadTs, limit, cursor, ct) => + Task.FromResult(new SlackNet.WebApi.ConversationMessagesResponse + { + Messages = + [ + new SlackNet.Events.MessageEvent + { + Ts = threadTs, + User = "U_ROOT", + Text = "thread root" + }, + new SlackNet.Events.MessageEvent + { + Ts = $"{threadTs[..^1]}1", + User = "U_ALICE", + Text = "historical doc", + Files = + [ + new SlackNet.File + { + Id = "F_DOCX", + Name = "notes.docx", + Mimetype = "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + Size = FakePngBytes.Length, + UrlPrivateDownload = "https://files.slack.com/fake/notes.docx" + } + ] + } + ] + }), + new SlackChannelOptions { BotToken = new SensitiveString("xoxb-fake") }, + httpClient, + new NullContentScanner(), + NullLogger.Instance); + + var profiles = ToolAudienceProfileDefaults.CreateProfiles(); + profiles.Public.ChannelAttachments = ToolAudienceProfileDefaults.CreatePublicChannelAttachments(); + + var deps = new SlackGatewayDependencies( + Pipeline: pipeline, + IngressGate: null, + ActorSystem: Sys, + TimeProvider: TimeProvider.System, + Options: new SlackChannelOptions + { + Enabled = true, + MentionOnly = true, + AllowedChannelIds = ["C_PUBLIC"], + ChannelAudiences = new Dictionary(StringComparer.Ordinal) + { + ["C_PUBLIC"] = "public" + }, + BotToken = new SensitiveString("xoxb-fake-token") + }, + BotUserId: new SlackUserId("UBOT"), + DefaultChannelId: null, + ReplyClient: _replyClient, + ContentScanner: new NullContentScanner(), + HttpClient: httpClient, + ThreadHistoryFetcher: fetcher, + AudienceProfiles: profiles, + ModelCapabilities: TestSlackGatewayDeps.DefaultVisionCapableModel, + Paths: _paths); + + var gateway = Sys.ActorOf(SlackGatewayActor.CreateProps(deps), "slack-gw-backfill-public-doc"); + + gateway.Tell(new SlackInboundMessage( + Kind: SlackInboundKind.AppMention, + EventId: new SlackEventId("C_PUBLIC:7000.2"), + ChannelId: new SlackChannelId("C_PUBLIC"), + ThreadTs: new SlackThreadTs("7000.0"), + EventTs: new SlackEventTs("7000.2"), + UserId: new SlackUserId("U_MENTIONER"), + BotId: null, + Text: "<@UBOT> please summarize", + Subtype: null, + Hidden: false, + IsDirectMessage: false)); + + await AwaitAssertAsync(() => + { + Assert.True(_chatClient.CallCount > 0, "Expected at least one LLM call"); + }, duration: TimeSpan.FromSeconds(10), cancellationToken: TestContext.Current.CancellationToken); + + var messages = _chatClient.LastMessages!; + var user = Assert.Single(messages, m => m.Role == AiChatRole.User); + + Assert.Empty(user.Contents.OfType()); + + var mergedText = string.Join("", user.Contents.OfType().Select(t => t.Text)); + Assert.Contains("attachment rejected", mergedText, StringComparison.OrdinalIgnoreCase); + Assert.Contains("category not allowed", mergedText, StringComparison.OrdinalIgnoreCase); + } + // --- Fake replies fetcher --- private Task FakeRepliesFetcher( diff --git a/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs b/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs index 8253db33a..c0bbd1a93 100644 --- a/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs +++ b/src/Netclaw.Channels.Slack/SlackThreadBindingActor.cs @@ -473,22 +473,18 @@ private async Task TryIngestSingleAttachmentAsync( if (!scanResult.IsAllowed) { + _log.Warning( + "slack_attachment_rejected name={Name} mime={Mime} reason=scan-blocked error={ScanError} message={ScanMessage}", + file.Name, file.MimeType, scanResult.Error?.ToString(), scanResult.Message ?? scanResult.Error?.ToString()); + if (scanResult.Error == ContentScanError.ScanFailure) { - // Scanner itself is broken — allow the file through rather than - // silently dropping it. The LLM provider will still validate. - _log.Error( - "Content scanner failed for file {Name}: {Message} — allowing file through", - file.Name, scanResult.Message); - } - else - { - _log.Warning( - "slack_attachment_rejected name={Name} mime={Mime} reason=scan-blocked message={ScanMessage}", - file.Name, file.MimeType, scanResult.Message ?? scanResult.Error?.ToString()); return new AttachmentIngestResult.Rejected( - $"Content scanner rejected `{file.Name}`: {scanResult.Message ?? scanResult.Error?.ToString()}."); + $"Couldn't scan `{file.Name}` — please try again later."); } + + return new AttachmentIngestResult.Rejected( + $"Content scanner rejected `{file.Name}`: {scanResult.Message ?? scanResult.Error?.ToString()}."); } // Write to inbox with filesystem-level collision suffixing and atomic move. @@ -794,7 +790,12 @@ private async Task BuildInputForInboundAsync( if (gap.Count == 0) return new InboundBuildResult(baseInput, detectorUnavailable); - var mergedContents = MergeGapWithLiveContents(gap, liveContents); + var mergedContents = MergeGapWithLiveContents( + gap, + liveContents, + triggeringMessage.Audience, + _dependencies.AudienceProfiles, + _dependencies.ModelCapabilities); return new InboundBuildResult(baseInput with { Contents = mergedContents }, detectorUnavailable); } @@ -875,12 +876,25 @@ private enum ClassificationOutcome private readonly record struct CursorAdvanced(string CursorTs); - private static List MergeGapWithLiveContents(IReadOnlyList gap, IReadOnlyList liveContents) + private static List MergeGapWithLiveContents( + IReadOnlyList gap, + IReadOnlyList liveContents, + TrustAudience audience, + ToolAudienceProfiles? audienceProfiles, + ModelCapabilities modelCapabilities) { + var profile = ToolAudienceProfileDefaults.GetResolvedProfile(audienceProfiles, audience); + var attachmentPolicy = profile.ChannelAttachments ?? ChannelAttachmentPolicy.Empty; + var inlineImages = modelCapabilities.InputModalities.HasFlag(ModelModality.Image); + var inlinePdfs = modelCapabilities.InputModalities.HasFlag(ModelModality.Image); + var sb = new StringBuilder(); sb.AppendLine("[thread history — messages exchanged before this inbound event]"); sb.AppendLine(); + var merged = new List(); + var acceptedBackfillData = new List(); + foreach (var item in gap) { var ts = item.ReceivedAt == default ? string.Empty : $", {item.ReceivedAt:yyyy-MM-dd HH:mm} UTC"; @@ -894,8 +908,36 @@ private static List MergeGapWithLiveContents(IReadOnlyList MergeGapWithLiveContents(IReadOnlyList { new TextContent(mergedText) }; - - // Gap image DataContent is carried through to the session so vision- - // capable models see the prior-thread images. The session actor's - // modality gate strips them when the resolved model doesn't support - // image input, so non-vision models are unaffected. - foreach (var item in gap) - { - foreach (var content in item.Contents) - { - if (content is not TextContent) - merged.Add(content); - } - } + merged.Add(new TextContent(mergedText)); + merged.AddRange(acceptedBackfillData); foreach (var content in liveContents) { diff --git a/src/Netclaw.Configuration.Tests/ChannelAttachmentPolicyTests.cs b/src/Netclaw.Configuration.Tests/ChannelAttachmentPolicyTests.cs index 594f56ce1..74c63c5df 100644 --- a/src/Netclaw.Configuration.Tests/ChannelAttachmentPolicyTests.cs +++ b/src/Netclaw.Configuration.Tests/ChannelAttachmentPolicyTests.cs @@ -134,7 +134,7 @@ public sealed class AttachmentCategoriesTests [InlineData("text/markdown", AttachmentCategory.Document)] [InlineData("application/json", AttachmentCategory.Document)] [InlineData("application/zip", AttachmentCategory.Archive)] - [InlineData("application/x-tar", AttachmentCategory.Archive)] + [InlineData("application/x-tar", AttachmentCategory.Other)] [InlineData("application/gzip", AttachmentCategory.Archive)] [InlineData("application/x-7z-compressed", AttachmentCategory.Archive)] [InlineData("video/mp4", AttachmentCategory.Media)] diff --git a/src/Netclaw.Configuration/ChannelAttachmentPolicy.cs b/src/Netclaw.Configuration/ChannelAttachmentPolicy.cs index 00913b65d..29ea0b7b2 100644 --- a/src/Netclaw.Configuration/ChannelAttachmentPolicy.cs +++ b/src/Netclaw.Configuration/ChannelAttachmentPolicy.cs @@ -105,12 +105,9 @@ public static AttachmentCategory FromMime(string? mime) { "application/zip" => true, "application/x-zip-compressed" => true, - "application/x-tar" => true, "application/gzip" => true, "application/x-gzip" => true, "application/x-7z-compressed" => true, - "application/x-rar-compressed" => true, - "application/vnd.rar" => true, "application/x-bzip2" => true, "application/x-xz" => true, _ => false diff --git a/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs b/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs index 572ae3e60..03255929a 100644 --- a/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs +++ b/src/Netclaw.Security.Tests/MagicByteValidatorTests.cs @@ -31,7 +31,6 @@ public sealed class MagicByteValidatorTests // ── Archive signatures ──────────────────────────────────────────────── private static readonly byte[] SevenZipHeader = [0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C, 0x00, 0x04]; - private static readonly byte[] RarHeader = [0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00]; private static readonly byte[] GzipHeader = [0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00]; // bzip2: "BZh" + block size '9' + 6-byte BCD-Pi compressed block header (31 41 59 26 53 59) private static readonly byte[] Bzip2Header = @@ -294,14 +293,6 @@ public void Validate_SevenZipAllowed() Assert.True(result.IsAllowed); } - [Fact] - public void Validate_RarAllowed() - { - var result = MagicByteValidator.Validate(RarHeader, "application/vnd.rar", "archive.rar"); - - Assert.True(result.IsAllowed); - } - [Fact] public void Validate_GzipAllowed() { @@ -723,28 +714,6 @@ public void Validate_RejectsId3WithBogusMajorVersion() Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); } - [Fact] - public void Validate_RejectsRarWithoutValidVariantTail() - { - // Rar!\x1A\x07\xFF — matches the looser 6-byte prefix, but the - // tightened check requires v4 (0x00) or v5 (0x01 0x00). - byte[] bogusRar = [0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0xFF]; - var result = MagicByteValidator.Validate(bogusRar, "application/vnd.rar", "bogus.rar"); - - Assert.False(result.IsAllowed); - Assert.Equal(ContentScanError.MimeTypeMismatch, result.Error); - } - - [Fact] - public void Validate_AcceptsRarV5WithEightByteMagic() - { - // RAR v5 signature: 8 bytes ending in 01 00. - byte[] rarV5 = [0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00, 0x00, 0x00]; - var result = MagicByteValidator.Validate(rarV5, "application/vnd.rar", "archive.rar"); - - Assert.True(result.IsAllowed); - } - [Fact] public void DetectMimeType_returns_null_for_unknown_content() { diff --git a/src/Netclaw.Security/MagicByteValidator.cs b/src/Netclaw.Security/MagicByteValidator.cs index 802f59fbd..0c8ffd866 100644 --- a/src/Netclaw.Security/MagicByteValidator.cs +++ b/src/Netclaw.Security/MagicByteValidator.cs @@ -148,7 +148,6 @@ public static ContentScanResult Validate( if (IsOle(content)) return "application/x-ole-compound-document"; if (IsRtf(content)) return "application/rtf"; if (Is7z(content)) return "application/x-7z-compressed"; - if (IsRar(content)) return "application/vnd.rar"; if (IsGzip(content)) return "application/gzip"; if (IsBzip2(content)) return "application/x-bzip2"; if (IsXz(content)) return "application/x-xz"; @@ -228,8 +227,6 @@ private static Dictionary BuildRules() rules["application/zip"] = new(Exts(".zip"), IsZip); rules["application/x-zip-compressed"] = new(Exts(".zip"), IsZip); rules["application/x-7z-compressed"] = new(Exts(".7z"), Is7z); - rules["application/vnd.rar"] = new(Exts(".rar"), IsRar); - rules["application/x-rar-compressed"] = new(Exts(".rar"), IsRar); rules["application/gzip"] = new(Exts(".gz", ".tgz"), IsGzip); rules["application/x-gzip"] = new(Exts(".gz", ".tgz"), IsGzip); rules["application/x-bzip2"] = new(Exts(".bz2"), IsBzip2); @@ -370,21 +367,6 @@ private static bool Is7z(ReadOnlySpan c) => c[0] == 0x37 && c[1] == 0x7A && c[2] == 0xBC && c[3] == 0xAF && c[4] == 0x27 && c[5] == 0x1C; - /// - /// RAR v1.5–v4 (Rar!\x1A\x07\x00, 7 bytes) or RAR v5+ - /// (Rar!\x1A\x07\x01\x00, 8 bytes). Tightened from the looser - /// 6-byte prefix to require the exact trailing bytes of one variant. - /// - private static bool IsRar(ReadOnlySpan c) - { - if (c.Length < 7) return false; - if (c[0] != 0x52 || c[1] != 0x61 || c[2] != 0x72 || c[3] != 0x21) return false; - if (c[4] != 0x1A || c[5] != 0x07) return false; - if (c[6] == 0x00) return true; // v4 - if (c[6] == 0x01 && c.Length >= 8 && c[7] == 0x00) return true; // v5 - return false; - } - /// /// Gzip: 1F 8B plus compression method 08 (DEFLATE). /// RFC 1952 reserves methods 0–7; only 8 is defined. Rejecting