Skip to content

Commit 8e55cae

Browse files
kriskkrisk
committed
feat(token-search): add customizable tokenizer with unicode-aware default
Token search previously hardcoded /\b\w+\b/g, which (a) split tokens with internal punctuation like node.js, c++, U.S.A apart, and (b) produced empty token lists for non-Latin scripts (CJK, Devanagari, Hebrew, etc.) — making token search effectively unavailable to East Asian users. Two changes: 1. New `tokenize` option accepting either RegExp or (text) => string[]. The function form lets users plug in Intl.Segmenter for word-level CJK segmentation. Function-form tokenizers are rejected by FuseWorker (cannot cross postMessage); regex-form passes through cleanly. 2. Default tokenizer changes from /\b\w+\b/g to /[\p{L}\p{M}\p{N}_]+/gu. The new default handles unicode letter, mark, and number runs out of the box — so CJK is indexed (was empty before) and combining marks on Devanagari, vowelized Hebrew/Arabic, NFD-normalized Latin stay attached to their base letter. Latin behavior is unchanged for ASCII alphanumerics + underscore. This is a behavior change for corpora containing characters outside [A-Za-z0-9_]; score distributions may differ. Targeted for 7.5.0 minor. Closes #821
1 parent ebde7cb commit 8e55cae

23 files changed

Lines changed: 618 additions & 41 deletions

‎dist/fuse-worker.cjs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ class FuseWorker {
5050
if (typeof options.getFn === 'function') {
5151
throw new Error(FUSE_WORKER_UNSUPPORTED_FN_OPTION('getFn'));
5252
}
53+
if (typeof options.tokenize === 'function') {
54+
throw new Error(FUSE_WORKER_UNSUPPORTED_FN_OPTION('tokenize'));
55+
}
5356
const keys = options.keys;
5457
if (Array.isArray(keys)) {
5558
for (let i = 0, len = keys.length; i < len; i += 1) {

‎dist/fuse-worker.mjs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ class FuseWorker {
4747
if (typeof options.getFn === 'function') {
4848
throw new Error(FUSE_WORKER_UNSUPPORTED_FN_OPTION('getFn'));
4949
}
50+
if (typeof options.tokenize === 'function') {
51+
throw new Error(FUSE_WORKER_UNSUPPORTED_FN_OPTION('tokenize'));
52+
}
5053
const keys = options.keys;
5154
if (Array.isArray(keys)) {
5255
for (let i = 0, len = keys.length; i < len; i += 1) {

‎dist/fuse.basic.cjs‎

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,7 @@ const FuzzyOptions = {
203203
const AdvancedOptions = {
204204
useExtendedSearch: false,
205205
useTokenSearch: false,
206+
tokenize: undefined,
206207
getFn: get,
207208
ignoreLocation: false,
208209
ignoreFieldNorm: false,
@@ -1023,11 +1024,43 @@ function format(results, docs, {
10231024
});
10241025
}
10251026

1026-
const WORD = /\b\w+\b/g;
1027+
// Includes \p{M} (Mark) so combining marks stay attached to their base
1028+
// letter — without it, scripts like Devanagari and NFD-normalized Latin
1029+
// shatter (e.g. 'हिन्दी' → ['ह','न','द'], 'café'.normalize('NFD') → ['cafe']).
1030+
const DEFAULT_TOKEN = /[\p{L}\p{M}\p{N}_]+/gu;
1031+
const warned = new WeakSet();
1032+
function warnNonGlobal(regex) {
1033+
if (!warned.has(regex)) {
1034+
warned.add(regex);
1035+
console.warn(`[Fuse] tokenize regex ${regex} lacks the global flag; only the ` + `first match per text will be returned. Add the 'g' flag.`);
1036+
}
1037+
}
1038+
function resolveTokenize(tokenize) {
1039+
if (typeof tokenize === 'function') {
1040+
let validated = false;
1041+
return text => {
1042+
const result = tokenize(text);
1043+
if (!validated) {
1044+
validated = true;
1045+
if (!Array.isArray(result) || result.some(t => typeof t !== 'string')) {
1046+
throw new Error(`[Fuse] tokenize function must return string[]; received ${Array.isArray(result) ? 'array containing non-strings' : typeof result}.`);
1047+
}
1048+
}
1049+
return result;
1050+
};
1051+
}
1052+
if (tokenize instanceof RegExp) {
1053+
if (!tokenize.global) warnNonGlobal(tokenize);
1054+
return text => text.match(tokenize) || [];
1055+
}
1056+
return text => text.match(DEFAULT_TOKEN) || [];
1057+
}
10271058
function createAnalyzer({
10281059
isCaseSensitive = false,
1029-
ignoreDiacritics = false
1060+
ignoreDiacritics = false,
1061+
tokenize
10301062
} = {}) {
1063+
const tokenizeFn = resolveTokenize(tokenize);
10311064
return {
10321065
tokenize(text) {
10331066
if (!isCaseSensitive) {
@@ -1036,7 +1069,7 @@ function createAnalyzer({
10361069
if (ignoreDiacritics) {
10371070
text = stripDiacritics(text);
10381071
}
1039-
return text.match(WORD) || [];
1072+
return tokenizeFn(text);
10401073
}
10411074
};
10421075
}
@@ -1211,7 +1244,8 @@ class Fuse {
12111244
if (this.options.useTokenSearch) {
12121245
const analyzer = createAnalyzer({
12131246
isCaseSensitive: this.options.isCaseSensitive,
1214-
ignoreDiacritics: this.options.ignoreDiacritics
1247+
ignoreDiacritics: this.options.ignoreDiacritics,
1248+
tokenize: this.options.tokenize
12151249
});
12161250
this._invertedIndex = buildInvertedIndex(this._myIndex.records, this._myIndex.keys.length, analyzer);
12171251
}
@@ -1227,7 +1261,8 @@ class Fuse {
12271261
const record = this._myIndex.records[this._myIndex.records.length - 1];
12281262
const analyzer = createAnalyzer({
12291263
isCaseSensitive: this.options.isCaseSensitive,
1230-
ignoreDiacritics: this.options.ignoreDiacritics
1264+
ignoreDiacritics: this.options.ignoreDiacritics,
1265+
tokenize: this.options.tokenize
12311266
});
12321267
addToInvertedIndex(this._invertedIndex, record, this._myIndex.keys.length, analyzer);
12331268
}

‎dist/fuse.basic.min.cjs‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎dist/fuse.basic.min.mjs‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎dist/fuse.basic.mjs‎

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,7 @@ const FuzzyOptions = {
201201
const AdvancedOptions = {
202202
useExtendedSearch: false,
203203
useTokenSearch: false,
204+
tokenize: undefined,
204205
getFn: get,
205206
ignoreLocation: false,
206207
ignoreFieldNorm: false,
@@ -1021,11 +1022,43 @@ function format(results, docs, {
10211022
});
10221023
}
10231024

1024-
const WORD = /\b\w+\b/g;
1025+
// Includes \p{M} (Mark) so combining marks stay attached to their base
1026+
// letter — without it, scripts like Devanagari and NFD-normalized Latin
1027+
// shatter (e.g. 'हिन्दी' → ['ह','न','द'], 'café'.normalize('NFD') → ['cafe']).
1028+
const DEFAULT_TOKEN = /[\p{L}\p{M}\p{N}_]+/gu;
1029+
const warned = new WeakSet();
1030+
function warnNonGlobal(regex) {
1031+
if (!warned.has(regex)) {
1032+
warned.add(regex);
1033+
console.warn(`[Fuse] tokenize regex ${regex} lacks the global flag; only the ` + `first match per text will be returned. Add the 'g' flag.`);
1034+
}
1035+
}
1036+
function resolveTokenize(tokenize) {
1037+
if (typeof tokenize === 'function') {
1038+
let validated = false;
1039+
return text => {
1040+
const result = tokenize(text);
1041+
if (!validated) {
1042+
validated = true;
1043+
if (!Array.isArray(result) || result.some(t => typeof t !== 'string')) {
1044+
throw new Error(`[Fuse] tokenize function must return string[]; received ${Array.isArray(result) ? 'array containing non-strings' : typeof result}.`);
1045+
}
1046+
}
1047+
return result;
1048+
};
1049+
}
1050+
if (tokenize instanceof RegExp) {
1051+
if (!tokenize.global) warnNonGlobal(tokenize);
1052+
return text => text.match(tokenize) || [];
1053+
}
1054+
return text => text.match(DEFAULT_TOKEN) || [];
1055+
}
10251056
function createAnalyzer({
10261057
isCaseSensitive = false,
1027-
ignoreDiacritics = false
1058+
ignoreDiacritics = false,
1059+
tokenize
10281060
} = {}) {
1061+
const tokenizeFn = resolveTokenize(tokenize);
10291062
return {
10301063
tokenize(text) {
10311064
if (!isCaseSensitive) {
@@ -1034,7 +1067,7 @@ function createAnalyzer({
10341067
if (ignoreDiacritics) {
10351068
text = stripDiacritics(text);
10361069
}
1037-
return text.match(WORD) || [];
1070+
return tokenizeFn(text);
10381071
}
10391072
};
10401073
}
@@ -1209,7 +1242,8 @@ class Fuse {
12091242
if (this.options.useTokenSearch) {
12101243
const analyzer = createAnalyzer({
12111244
isCaseSensitive: this.options.isCaseSensitive,
1212-
ignoreDiacritics: this.options.ignoreDiacritics
1245+
ignoreDiacritics: this.options.ignoreDiacritics,
1246+
tokenize: this.options.tokenize
12131247
});
12141248
this._invertedIndex = buildInvertedIndex(this._myIndex.records, this._myIndex.keys.length, analyzer);
12151249
}
@@ -1225,7 +1259,8 @@ class Fuse {
12251259
const record = this._myIndex.records[this._myIndex.records.length - 1];
12261260
const analyzer = createAnalyzer({
12271261
isCaseSensitive: this.options.isCaseSensitive,
1228-
ignoreDiacritics: this.options.ignoreDiacritics
1262+
ignoreDiacritics: this.options.ignoreDiacritics,
1263+
tokenize: this.options.tokenize
12291264
});
12301265
addToInvertedIndex(this._invertedIndex, record, this._myIndex.keys.length, analyzer);
12311266
}

‎dist/fuse.cjs‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ const FuzzyOptions = {
200200
const AdvancedOptions = {
201201
useExtendedSearch: false,
202202
useTokenSearch: false,
203+
tokenize: undefined,
203204
getFn: get,
204205
ignoreLocation: false,
205206
ignoreFieldNorm: false,
@@ -1418,11 +1419,43 @@ function format(results, docs, {
14181419
});
14191420
}
14201421

1421-
const WORD = /\b\w+\b/g;
1422+
// Includes \p{M} (Mark) so combining marks stay attached to their base
1423+
// letter — without it, scripts like Devanagari and NFD-normalized Latin
1424+
// shatter (e.g. 'हिन्दी' → ['ह','न','द'], 'café'.normalize('NFD') → ['cafe']).
1425+
const DEFAULT_TOKEN = /[\p{L}\p{M}\p{N}_]+/gu;
1426+
const warned = new WeakSet();
1427+
function warnNonGlobal(regex) {
1428+
if (!warned.has(regex)) {
1429+
warned.add(regex);
1430+
console.warn(`[Fuse] tokenize regex ${regex} lacks the global flag; only the ` + `first match per text will be returned. Add the 'g' flag.`);
1431+
}
1432+
}
1433+
function resolveTokenize(tokenize) {
1434+
if (typeof tokenize === 'function') {
1435+
let validated = false;
1436+
return text => {
1437+
const result = tokenize(text);
1438+
if (!validated) {
1439+
validated = true;
1440+
if (!Array.isArray(result) || result.some(t => typeof t !== 'string')) {
1441+
throw new Error(`[Fuse] tokenize function must return string[]; received ${Array.isArray(result) ? 'array containing non-strings' : typeof result}.`);
1442+
}
1443+
}
1444+
return result;
1445+
};
1446+
}
1447+
if (tokenize instanceof RegExp) {
1448+
if (!tokenize.global) warnNonGlobal(tokenize);
1449+
return text => text.match(tokenize) || [];
1450+
}
1451+
return text => text.match(DEFAULT_TOKEN) || [];
1452+
}
14221453
function createAnalyzer({
14231454
isCaseSensitive = false,
1424-
ignoreDiacritics = false
1455+
ignoreDiacritics = false,
1456+
tokenize
14251457
} = {}) {
1458+
const tokenizeFn = resolveTokenize(tokenize);
14261459
return {
14271460
tokenize(text) {
14281461
if (!isCaseSensitive) {
@@ -1431,7 +1464,7 @@ function createAnalyzer({
14311464
if (ignoreDiacritics) {
14321465
text = stripDiacritics(text);
14331466
}
1434-
return text.match(WORD) || [];
1467+
return tokenizeFn(text);
14351468
}
14361469
};
14371470
}
@@ -1602,7 +1635,8 @@ class Fuse {
16021635
if (this.options.useTokenSearch) {
16031636
const analyzer = createAnalyzer({
16041637
isCaseSensitive: this.options.isCaseSensitive,
1605-
ignoreDiacritics: this.options.ignoreDiacritics
1638+
ignoreDiacritics: this.options.ignoreDiacritics,
1639+
tokenize: this.options.tokenize
16061640
});
16071641
this._invertedIndex = buildInvertedIndex(this._myIndex.records, this._myIndex.keys.length, analyzer);
16081642
}
@@ -1618,7 +1652,8 @@ class Fuse {
16181652
const record = this._myIndex.records[this._myIndex.records.length - 1];
16191653
const analyzer = createAnalyzer({
16201654
isCaseSensitive: this.options.isCaseSensitive,
1621-
ignoreDiacritics: this.options.ignoreDiacritics
1655+
ignoreDiacritics: this.options.ignoreDiacritics,
1656+
tokenize: this.options.tokenize
16221657
});
16231658
addToInvertedIndex(this._invertedIndex, record, this._myIndex.keys.length, analyzer);
16241659
}
@@ -1990,7 +2025,8 @@ class TokenSearch {
19902025
this.options = options;
19912026
this.analyzer = createAnalyzer({
19922027
isCaseSensitive: options.isCaseSensitive,
1993-
ignoreDiacritics: options.ignoreDiacritics
2028+
ignoreDiacritics: options.ignoreDiacritics,
2029+
tokenize: options.tokenize
19942030
});
19952031
const queryTerms = this.analyzer.tokenize(pattern);
19962032
const invertedIndex = options._invertedIndex;

‎dist/fuse.d.ts‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@ interface KeyObject {
2525
}
2626
type FuseGetFunction<T> = (obj: T, path: string | string[]) => ReadonlyArray<string> | string;
2727
type GetFunction = (obj: any, path: string | string[]) => any;
28+
/**
29+
* Custom tokenizer for `useTokenSearch`. Receives the field/query text after
30+
* case-folding and diacritic-stripping (per `isCaseSensitive` /
31+
* `ignoreDiacritics`) and must return the term list. Functions must be
32+
* deterministic — non-deterministic output silently breaks `df` accounting.
33+
*/
34+
type FuseTokenizeFunction = (text: string) => string[];
2835
interface NormInterface {
2936
get(value: string): number;
3037
clear(): void;
@@ -142,6 +149,15 @@ interface IFuseOptions<T> {
142149
useExtendedSearch?: boolean;
143150
/** When `true`, enables token search with TF-IDF scoring. */
144151
useTokenSearch?: boolean;
152+
/**
153+
* Tokenizer used by `useTokenSearch`, applied identically at index-build
154+
* and query time. Accepts either a global `RegExp` or a function returning
155+
* `string[]`. Defaults to `/[\p{L}\p{M}\p{N}_]+/gu`, which handles CJK,
156+
* Cyrillic, Greek, Arabic, Hebrew, Devanagari, etc. out of the box. Use a
157+
* function form (e.g. wrapping `Intl.Segmenter`) for word-segmentation in
158+
* scripts without whitespace boundaries.
159+
*/
160+
tokenize?: RegExp | FuseTokenizeFunction;
145161
}
146162
interface FuseIndexOptions<T> {
147163
getFn?: FuseGetFunction<T>;
@@ -289,4 +305,4 @@ declare class Fuse<T> {
289305
}
290306

291307
export { FuseIndex, Fuse as default };
292-
export type { Expression, FuseGetFunction, FuseIndexObjectRecord, FuseIndexOptions, FuseIndexRecords, FuseIndexStringRecord, FuseOptionKey, FuseOptionKeyObject, FuseResult, FuseResultMatch, FuseSearchOptions, FuseSortFunction, FuseSortFunctionArg, FuseSortFunctionItem, FuseSortFunctionMatch, FuseSortFunctionMatchList, IFuseOptions, RangeTuple, RecordEntry, RecordEntryArrayItem, RecordEntryObject, SearchResult };
308+
export type { Expression, FuseGetFunction, FuseIndexObjectRecord, FuseIndexOptions, FuseIndexRecords, FuseIndexStringRecord, FuseOptionKey, FuseOptionKeyObject, FuseResult, FuseResultMatch, FuseSearchOptions, FuseSortFunction, FuseSortFunctionArg, FuseSortFunctionItem, FuseSortFunctionMatch, FuseSortFunctionMatchList, FuseTokenizeFunction, IFuseOptions, RangeTuple, RecordEntry, RecordEntryArrayItem, RecordEntryObject, SearchResult };

‎dist/fuse.min.cjs‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎dist/fuse.min.mjs‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)