From ff5cbd9f101d5d02065f41a03dcd1008958d1ada Mon Sep 17 00:00:00 2001 From: Zhang Sheng Date: Mon, 20 Jul 2026 13:44:50 +0800 Subject: [PATCH] fix: improve Chinese NLP filetype detection accuracy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Added new test case for design source files shadowing image substrings 2. Included "包" back to application filetype rules with negative lookahead to prevent false matches 3. Increased priority of design source files to shadow image matches 4. Removed outdated comment about span consumption since it's now standard behavior Influence: 1. Test Chinese file search with "矢量图" to ensure design source files are prioritized over general image files 2. Verify "包" matches application filetype while avoiding false positives like "表情包" 3. Check that design source files (PSD, AI etc.) are correctly identified in complex queries fix: 提升中文NLP文件类型检测准确性 1. 新增设计源文件遮蔽图片子字符串的测试用例 2. 重新包含"包"到应用文件类型规则，添加否定前瞻避免误判 3. 提高设计源文件的优先级使其遮蔽图片匹配 4. 移除关于span消费的过时注释，因这已是标准行为 Influence: 1. 测试中文文件搜索"矢量图"确保设计源文件优先于普通图片文件 2. 验证"包"能匹配应用文件类型同时避免"表情包"等误判 3. 检查在复杂查询中设计源文件(PSD, AI等)能被正确识别 Fixes: #370659 --- .../dfm-search-tests/tst_chinese_nlp.cpp | 26 +++++++++++++++++-- .../semantic/extractors/filetypeextractor.cpp | 1 - .../semantic/rules/zh_CN/filetype_rules.json | 4 +-- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/autotests/dfm-search-tests/tst_chinese_nlp.cpp b/autotests/dfm-search-tests/tst_chinese_nlp.cpp index d845c36e..7f382bc0 100644 --- a/autotests/dfm-search-tests/tst_chinese_nlp.cpp +++ b/autotests/dfm-search-tests/tst_chinese_nlp.cpp @@ -168,6 +168,7 @@ private Q_SLOTS: void fileType_archive_allSynonyms(); void fileType_application_allSynonyms(); void fileType_design_source_allSynonyms(); + void fileType_designSource_shadowsImageSubstring(); // Filetype specific+general combination tests (span leak regression) void fileType_specificAndGeneral_pptDocument(); @@ -1287,10 +1288,9 @@ void tst_ChineseNLP::fileType_archive_allSynonyms() void tst_ChineseNLP::fileType_application_allSynonyms() { // Requirements: 安装包, 软件, 应用, 脚本, 程序, 包 - // NOTE: "包" excluded from rules to avoid false positives with "表情包", "压缩包" const QStringList inputs = { QStringLiteral("安装包"), QStringLiteral("软件"), QStringLiteral("应用"), - QStringLiteral("脚本"), QStringLiteral("程序") + QStringLiteral("脚本"), QStringLiteral("程序"), QStringLiteral("包") }; for (const QString &input : inputs) { ParsedIntent intent; @@ -1320,6 +1320,28 @@ void tst_ChineseNLP::fileType_design_source_allSynonyms() } } +void tst_ChineseNLP::fileType_designSource_shadowsImageSubstring() +{ + ParsedIntent intent; + m_parser->parse(QStringLiteral("昨天创建的矢量图"), intent); + + QVERIFY2(setEquals(intent.fileExtensions(), expectedExtsForRule("filetype_design_source")), + qPrintable(QStringLiteral("Extensions: ") + intent.fileExtensions().join(','))); + + bool hasDesignSource = false; + bool hasImage = false; + for (const MatchSpan &span : intent.consumedSpans()) { + if (span.ruleId() == QLatin1String("filetype_design_source")) { + hasDesignSource = true; + } else if (span.ruleId() == QLatin1String("filetype_image")) { + hasImage = true; + } + } + + QVERIFY(hasDesignSource); + QVERIFY(!hasImage); +} + // ===== Filetype specific+general combination tests (span leak regression) ===== // 当 specific filetype 规则与 general filetype 规则同时命中时，general 规则匹配的 // 文本（文档/表格/幻灯片）必须被消费，不得泄漏成 keyword；且 fileExtensions 应只 diff --git a/src/dfm-search/dfm-search-lib/semantic/extractors/filetypeextractor.cpp b/src/dfm-search/dfm-search-lib/semantic/extractors/filetypeextractor.cpp index f91205e5..b50fe8fb 100644 --- a/src/dfm-search/dfm-search-lib/semantic/extractors/filetypeextractor.cpp +++ b/src/dfm-search/dfm-search-lib/semantic/extractors/filetypeextractor.cpp @@ -48,7 +48,6 @@ void FileTypeExtractor::extract(const QString &input, ParsedIntent &intent) continue; } - // Always consume the matched span so the matched text doesn't leak into keywords MatchSpan span; span.setStart(m.capturedStart()); span.setEnd(m.capturedEnd()); diff --git a/src/dfm-search/dfm-search-lib/semantic/rules/zh_CN/filetype_rules.json b/src/dfm-search/dfm-search-lib/semantic/rules/zh_CN/filetype_rules.json index f52acfb5..237ed23b 100644 --- a/src/dfm-search/dfm-search-lib/semantic/rules/zh_CN/filetype_rules.json +++ b/src/dfm-search/dfm-search-lib/semantic/rules/zh_CN/filetype_rules.json @@ -129,7 +129,7 @@ }, { "id": "filetype_application", - "pattern": "安装包|软件|应用|脚本|程序|包", + "pattern": "安装包|软件|应用|脚本|程序|包(?!含)", "description": "Application packages", "enabled": true, "priority": 150, @@ -143,7 +143,7 @@ "pattern": "源文件|设计稿|矢量图|工程文件|psd|fig|sketch", "description": "Design source files", "enabled": true, - "priority": 150, + "priority": 160, "metadata": { "extensions": ["psd", "ai", "fig", "sketch"] }