From 133585d536e2b1a7e8a7db4712e73b2c1a305aef Mon Sep 17 00:00:00 2001 From: Kuchizu Date: Wed, 1 Jul 2026 12:40:23 +0300 Subject: [PATCH 1/5] Trim event title to a maximum length in grouper --- workers/grouper/src/index.ts | 14 +++++++++++++- workers/grouper/tests/index.test.ts | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 8203e8d3..a0837eff 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -59,10 +59,15 @@ const DB_DUPLICATE_KEY_ERROR = '11000'; const DAILY_METRICS_RETENTION_DAYS = 90; /** - * Maximum length for backtrace code line or title + * Maximum length for backtrace code line */ const MAX_CODE_LINE_LENGTH = 140; +/** + * Maximum length for event title + */ +const MAX_TITLE_LENGTH = 1000; + /** * Worker for handling Javascript events */ @@ -209,6 +214,13 @@ export default class GrouperWorker extends Worker { }; } + /** + * Trim title before hashing so hash and stored title stay consistent + */ + if (typeof task.payload.title === 'string') { + task.payload.title = rightTrim(task.payload.title, MAX_TITLE_LENGTH); + } + let uniqueEventHash = await session.measureStep('hash', () => this.getUniqueEventHash(task)); let existedEvent: GroupedEventDBScheme; let repetitionId = null; diff --git a/workers/grouper/tests/index.test.ts b/workers/grouper/tests/index.test.ts index c75351a9..18a7dd36 100644 --- a/workers/grouper/tests/index.test.ts +++ b/workers/grouper/tests/index.test.ts @@ -173,6 +173,20 @@ describe('GrouperWorker', () => { expect(await eventsCollection.find().count()).toBe(1); }); + test('Should trim event title to the maximum length before saving', async () => { + const longTitle = 'A'.repeat(5000); + + await worker.handle(generateTask({ title: longTitle })); + + const savedEvent = await eventsCollection.findOne({}); + + /** + * 1000 chars + ellipsis + */ + expect(savedEvent.payload.title.length).toBe(1001); + expect(savedEvent.payload.title.endsWith('…')).toBe(true); + }); + test('Should increment total events count on each processing', async () => { await worker.handle(generateTask()); await worker.handle(generateTask()); From b9fd948486e971a7a88bdc983f8457d1d2e51a02 Mon Sep 17 00:00:00 2001 From: Kuchizu Date: Wed, 1 Jul 2026 21:11:06 +0300 Subject: [PATCH 2/5] refactor(grouper): move title trimming into DataFilter --- workers/grouper/src/data-filter.ts | 20 ++++++++++++++++++++ workers/grouper/src/index.ts | 9 +-------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/workers/grouper/src/data-filter.ts b/workers/grouper/src/data-filter.ts index 2345b8e5..d88b24e3 100644 --- a/workers/grouper/src/data-filter.ts +++ b/workers/grouper/src/data-filter.ts @@ -1,11 +1,17 @@ import type { EventAddons, EventData } from '@hawk.so/types'; import { unsafeFields } from '../../../lib/utils/unsafeFields'; +import { rightTrim } from '../../../lib/utils/string'; /** * Maximum depth for object traversal to prevent excessive memory allocations */ const MAX_TRAVERSAL_DEPTH = 20; +/** + * Maximum length for event title + */ +const MAX_TITLE_LENGTH = 1000; + /** * Recursively iterate through object and call function on each key * @@ -142,6 +148,20 @@ export default class DataFilter { }); } + /** + * Trim event title to the maximum allowed length. + * It mutates the original object. + * + * Should be called before hashing so the hash and the stored title stay consistent. + * + * @param event - event to process + */ + public trimEventTitle(event: EventData): void { + if (typeof event.title === 'string') { + event.title = rightTrim(event.title, MAX_TITLE_LENGTH); + } + } + /** * Recursively iterates object and applies filtering to its entries * diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index a0837eff..845c5590 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -63,11 +63,6 @@ const DAILY_METRICS_RETENTION_DAYS = 90; */ const MAX_CODE_LINE_LENGTH = 140; -/** - * Maximum length for event title - */ -const MAX_TITLE_LENGTH = 1000; - /** * Worker for handling Javascript events */ @@ -217,9 +212,7 @@ export default class GrouperWorker extends Worker { /** * Trim title before hashing so hash and stored title stay consistent */ - if (typeof task.payload.title === 'string') { - task.payload.title = rightTrim(task.payload.title, MAX_TITLE_LENGTH); - } + this.dataFilter.trimEventTitle(task.payload); let uniqueEventHash = await session.measureStep('hash', () => this.getUniqueEventHash(task)); let existedEvent: GroupedEventDBScheme; From 289bf6048511abd0a304346fd5d7189ddf996e7e Mon Sep 17 00:00:00 2001 From: Kuchizu Date: Wed, 8 Jul 2026 11:28:08 +0300 Subject: [PATCH 3/5] Trim grouper event titles --- workers/grouper/src/data-filter.ts | 8 ++++---- workers/grouper/src/index.ts | 13 ++++--------- workers/grouper/tests/index.test.ts | 4 ++-- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/workers/grouper/src/data-filter.ts b/workers/grouper/src/data-filter.ts index d88b24e3..06936883 100644 --- a/workers/grouper/src/data-filter.ts +++ b/workers/grouper/src/data-filter.ts @@ -8,9 +8,9 @@ import { rightTrim } from '../../../lib/utils/string'; const MAX_TRAVERSAL_DEPTH = 20; /** - * Maximum length for event title + * Maximum length for event title before appending ellipsis */ -const MAX_TITLE_LENGTH = 1000; +const MAX_TITLE_LENGTH = 400; /** * Recursively iterate through object and call function on each key @@ -141,6 +141,8 @@ export default class DataFilter { * @param event - event to process */ public processEvent(event: EventData): void { + this.trimEventTitle(event); + unsafeFields.forEach(field => { if (event[field]) { this.processField(event[field]); @@ -152,8 +154,6 @@ export default class DataFilter { * Trim event title to the maximum allowed length. * It mutates the original object. * - * Should be called before hashing so the hash and the stored title stay consistent. - * * @param event - event to process */ public trimEventTitle(event: EventData): void { diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 845c5590..58115340 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -198,8 +198,6 @@ export default class GrouperWorker extends Worker { this.grouperMetrics.observePayloadSize(taskPayloadSize); this.memoryMonitor.logBeforeHandle(memoryBeforeHandle, handledTasksCount, taskPayloadSize, task.projectId); - this.logger.info(`[handle] project=${task.projectId} catcher=${task.catcherType} title="${task.payload.title}" payloadSize=${taskPayloadSize}b backtraceFrames=${task.payload.backtrace?.length ?? 0}`); - // FIX RELEASE TYPE // TODO: REMOVE AFTER 01.01.2026, after the most of the users update to new js catcher if (task.payload && task.payload.release !== undefined) { @@ -210,9 +208,11 @@ export default class GrouperWorker extends Worker { } /** - * Trim title before hashing so hash and stored title stay consistent + * Filter event data before logging and hashing so logs, hash and stored event stay consistent. */ - this.dataFilter.trimEventTitle(task.payload); + this.dataFilter.processEvent(task.payload); + + this.logger.info(`[handle] project=${task.projectId} catcher=${task.catcherType} title="${task.payload.title}" payloadSize=${taskPayloadSize}b backtraceFrames=${task.payload.backtrace?.length ?? 0}`); let uniqueEventHash = await session.measureStep('hash', () => this.getUniqueEventHash(task)); let existedEvent: GroupedEventDBScheme; @@ -224,11 +224,6 @@ export default class GrouperWorker extends Worker { * Trim source code lines to prevent memory leaks */ this.trimSourceCodeLines(task.payload); - - /** - * Filter sensitive information - */ - this.dataFilter.processEvent(task.payload); }); /** diff --git a/workers/grouper/tests/index.test.ts b/workers/grouper/tests/index.test.ts index 18a7dd36..e1761bc5 100644 --- a/workers/grouper/tests/index.test.ts +++ b/workers/grouper/tests/index.test.ts @@ -181,9 +181,9 @@ describe('GrouperWorker', () => { const savedEvent = await eventsCollection.findOne({}); /** - * 1000 chars + ellipsis + * 400 chars + ellipsis */ - expect(savedEvent.payload.title.length).toBe(1001); + expect(savedEvent.payload.title.length).toBe(401); expect(savedEvent.payload.title.endsWith('…')).toBe(true); }); From 1157fd4a6657e6128253f857799c52c5b7f6db10 Mon Sep 17 00:00:00 2001 From: Kuchizu Date: Mon, 13 Jul 2026 13:02:25 +0300 Subject: [PATCH 4/5] Add Sanitizer for event payload fields and drop per-event handle log --- lib/utils/sanitizer.ts | 122 +++++++++++++++ workers/grouper/src/data-filter.ts | 32 ++++ workers/grouper/src/index.ts | 4 +- workers/grouper/tests/data-filter.test.ts | 182 +++++++++++++++++++--- 4 files changed, 313 insertions(+), 27 deletions(-) create mode 100644 lib/utils/sanitizer.ts diff --git a/lib/utils/sanitizer.ts b/lib/utils/sanitizer.ts new file mode 100644 index 00000000..dd79617c --- /dev/null +++ b/lib/utils/sanitizer.ts @@ -0,0 +1,122 @@ +import { rightTrim } from './string'; + +/** + * Maximum string length before appending ellipsis + */ +const MAX_STRING_LENGTH = 200; + +/** + * Objects with more keys are represented as "" + */ +const MAX_OBJECT_KEYS_COUNT = 20; + +/** + * Maximum depth of sanitized objects + */ +const MAX_DEPTH = 5; + +/** + * Maximum length of sanitized arrays + */ +const MAX_ARRAY_LENGTH = 10; + +/** + * Checks that the value is a plain object + * + * @param target - value to check + */ +function isPlainObject(target: unknown): target is Record { + return Object.prototype.toString.call(target) === '[object Object]'; +} + +/** + * Values that can be tracked by WeakSet to detect circular references + */ +type ObjectLike = Record | unknown[]; + +/** + * Prepares event data for storing: trims long strings, slices long arrays, + * replaces too deep/big objects and circular references with placeholders. + */ +export class Sanitizer { + /** + * Apply sanitizing for array/object/primitives + * + * @param data - any value to sanitize + * @param depth - current depth of recursion + * @param seen - already visited objects + */ + public static sanitize(data: unknown, depth = 0, seen = new WeakSet()): unknown { + if (data !== null && typeof data === 'object') { + if (seen.has(data as ObjectLike)) { + return ''; + } + seen.add(data as ObjectLike); + } + + if (Array.isArray(data)) { + return Sanitizer.sanitizeArray(data, depth + 1, seen); + } + + if (isPlainObject(data)) { + return Sanitizer.sanitizeObject(data, depth + 1, seen); + } + + if (typeof data === 'string') { + return rightTrim(data, MAX_STRING_LENGTH); + } + + return data; + } + + /** + * Slices array to the maximum length and sanitizes each element + * + * @param arr - array to sanitize + * @param depth - current depth of recursion + * @param seen - already visited objects + */ + private static sanitizeArray(arr: unknown[], depth: number, seen: WeakSet): unknown[] { + const length = arr.length; + + if (length > MAX_ARRAY_LENGTH) { + arr = arr.slice(0, MAX_ARRAY_LENGTH); + arr.push(`<${length - MAX_ARRAY_LENGTH} more items...>`); + } + + return arr.map((item) => { + return Sanitizer.sanitize(item, depth, seen); + }); + } + + /** + * Sanitizes object values recursively + * + * @param data - object to sanitize + * @param depth - current depth of recursion + * @param seen - already visited objects + */ + private static sanitizeObject( + data: Record, + depth: number, + seen: WeakSet + ): Record | '' | '' { + if (depth > MAX_DEPTH) { + return ''; + } + + if (Object.keys(data).length > MAX_OBJECT_KEYS_COUNT) { + return ''; + } + + const result: Record = {}; + + for (const key in data) { + if (Object.prototype.hasOwnProperty.call(data, key)) { + result[key] = Sanitizer.sanitize(data[key], depth, seen); + } + } + + return result; + } +} diff --git a/workers/grouper/src/data-filter.ts b/workers/grouper/src/data-filter.ts index 06936883..bd89d175 100644 --- a/workers/grouper/src/data-filter.ts +++ b/workers/grouper/src/data-filter.ts @@ -1,6 +1,7 @@ import type { EventAddons, EventData } from '@hawk.so/types'; import { unsafeFields } from '../../../lib/utils/unsafeFields'; import { rightTrim } from '../../../lib/utils/string'; +import { Sanitizer } from '../../../lib/utils/sanitizer'; /** * Maximum depth for object traversal to prevent excessive memory allocations @@ -142,6 +143,7 @@ export default class DataFilter { */ public processEvent(event: EventData): void { this.trimEventTitle(event); + this.sanitizeEvent(event); unsafeFields.forEach(field => { if (event[field]) { @@ -162,6 +164,36 @@ export default class DataFilter { } } + /** + * Sanitize event fields that can contain long strings, deep objects or long arrays. + * It mutates the original object. + * + * @param event - event to process + */ + public sanitizeEvent(event: EventData): void { + unsafeFields.forEach(field => { + if (event[field] !== undefined) { + (event as unknown as Record)[field] = Sanitizer.sanitize(event[field]); + } + }); + + event.backtrace?.forEach(frame => { + if (frame.arguments !== undefined) { + frame.arguments = Sanitizer.sanitize(frame.arguments) as string[]; + } + }); + + event.breadcrumbs?.forEach(breadcrumb => { + if (typeof breadcrumb.message === 'string') { + breadcrumb.message = Sanitizer.sanitize(breadcrumb.message) as string; + } + + if (breadcrumb.data !== undefined) { + breadcrumb.data = Sanitizer.sanitize(breadcrumb.data) as typeof breadcrumb.data; + } + }); + } + /** * Recursively iterates object and applies filtering to its entries * diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 58115340..46f77c98 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -208,12 +208,10 @@ export default class GrouperWorker extends Worker { } /** - * Filter event data before logging and hashing so logs, hash and stored event stay consistent. + * Filter event data before hashing so hash and stored event stay consistent. */ this.dataFilter.processEvent(task.payload); - this.logger.info(`[handle] project=${task.projectId} catcher=${task.catcherType} title="${task.payload.title}" payloadSize=${taskPayloadSize}b backtraceFrames=${task.payload.backtrace?.length ?? 0}`); - let uniqueEventHash = await session.measureStep('hash', () => this.getUniqueEventHash(task)); let existedEvent: GroupedEventDBScheme; let repetitionId = null; diff --git a/workers/grouper/tests/data-filter.test.ts b/workers/grouper/tests/data-filter.test.ts index 2f00dd68..d472244b 100644 --- a/workers/grouper/tests/data-filter.test.ts +++ b/workers/grouper/tests/data-filter.test.ts @@ -14,16 +14,24 @@ jest.mock('amqplib'); * @param [options.context] - generated event context * @param [options.addons] - generated event addons */ -function generateEvent({ context, addons }: {context?: Json, addons?: EventAddons}): EventData { +function generateEvent({ context, addons, backtrace, breadcrumbs }: { + context?: Json, + addons?: EventAddons, + backtrace?: EventData['backtrace'], + breadcrumbs?: EventData['breadcrumbs'], +}): EventData { return { title: 'Event with sensitive data', - backtrace: [], + backtrace: backtrace ?? [], ...(context && { context, }), ...(addons && { addons, }), + ...(breadcrumbs && { + breadcrumbs, + }), }; } @@ -161,30 +169,52 @@ describe('GrouperWorker', () => { }); test('should filter additional sensitive keys (authorization, token, payment, dsn, ssn, etc.) in context', async () => { + /** + * Split the mock into two groups to keep objects under the sanitizer keys limit + */ + const entries = Object.entries(additionalSensitiveDataMock); + const half = Math.ceil(entries.length / 2); const event = generateEvent({ - context: additionalSensitiveDataMock, + context: { + group1: Object.fromEntries(entries.slice(0, half)), + group2: Object.fromEntries(entries.slice(half)), + }, }); dataFilter.processEvent(event); - Object.keys(additionalSensitiveDataMock).forEach((key) => { - expect(event.context[key]).toBe('[filtered]'); + entries.slice(0, half).forEach(([ key ]) => { + expect(event.context['group1'][key]).toBe('[filtered]'); + }); + entries.slice(half).forEach(([ key ]) => { + expect(event.context['group2'][key]).toBe('[filtered]'); }); }); test('should filter additional sensitive keys in addons', async () => { + /** + * Split the mock into two groups to keep objects under the sanitizer keys limit + */ + const entries = Object.entries(additionalSensitiveDataMock); + const half = Math.ceil(entries.length / 2); const event = generateEvent({ addons: { vue: { - props: additionalSensitiveDataMock, + props: { + group1: Object.fromEntries(entries.slice(0, half)), + group2: Object.fromEntries(entries.slice(half)), + }, }, }, }); dataFilter.processEvent(event); - Object.keys(additionalSensitiveDataMock).forEach((key) => { - expect(event.addons['vue']['props'][key]).toBe('[filtered]'); + entries.slice(0, half).forEach(([ key ]) => { + expect(event.addons['vue']['props']['group1'][key]).toBe('[filtered]'); + }); + entries.slice(half).forEach(([ key ]) => { + expect(event.addons['vue']['props']['group2'][key]).toBe('[filtered]'); }); }); @@ -328,9 +358,9 @@ describe('GrouperWorker', () => { expect(event.context['auth']).toBe('[filtered]'); }); - test('should handle deeply nested objects (>20 levels) without excessive memory allocations', () => { - // Create an object nested deeper than the cap (>20 levels) - let deeplyNested: any = { value: 'leaf', secret: 'should-be-filtered' }; + test('should replace too deep objects with a placeholder and keep filtering reachable levels', () => { + // Create an object nested deeper than the sanitizer depth cap + let deeplyNested: any = { value: 'leaf', secret: 'should-be-cut-off' }; for (let i = 0; i < 25; i++) { deeplyNested = { [`level${i}`]: deeplyNested, password: `sensitive${i}` }; @@ -343,23 +373,127 @@ describe('GrouperWorker', () => { // This should not throw or cause memory issues dataFilter.processEvent(event); - // Verify that filtering still works at various depths + // Filtering still works on the levels kept by the sanitizer expect(event.context['password']).toBe('[filtered]'); - // Navigate to a mid-level and check filtering - let current = event.context['level24'] as any; - for (let i = 24; i > 15; i--) { - expect(current['password']).toBe('[filtered]'); - current = current[`level${i - 1}`]; - } + const deepestKeptLevel = event.context['level24']['level23']['level22']['level21']; + + expect(deepestKeptLevel['password']).toBe('[filtered]'); + + // Everything deeper is replaced with a placeholder + expect(deepestKeptLevel['level20']).toBe(''); + }); + }); + + describe('Sanitizer', () => { + test('should trim long strings in context', () => { + const longString = 'a'.repeat(5000); + const event = generateEvent({ + context: { + longValue: longString, + }, + }); + + dataFilter.processEvent(event); + + expect(event.context['longValue']).toBe('a'.repeat(200) + '…'); + }); - // At the leaf level, the secret should still be filtered - // (though path tracking may be capped, filtering should still work) - let leaf = event.context; - for (let i = 24; i >= 0; i--) { - leaf = leaf[`level${i}`] as any; + test('should replace objects with too many keys with a placeholder', () => { + const bigObject: Record = {}; + + for (let i = 0; i < 25; i++) { + bigObject[`key${i}`] = i; } - expect(leaf['secret']).toBe('[filtered]'); + + const event = generateEvent({ + context: { + bigObject, + }, + }); + + dataFilter.processEvent(event); + + expect(event.context['bigObject']).toBe(''); + }); + + test('should slice long arrays and add a placeholder', () => { + const event = generateEvent({ + context: { + longArray: new Array(15).fill('item') as unknown as Json, + }, + }); + + dataFilter.processEvent(event); + + const sanitizedArray = event.context['longArray']; + + expect(sanitizedArray).toHaveLength(11); // 10 items + placeholder + expect(sanitizedArray[10]).toBe('<5 more items...>'); + }); + + test('should sanitize addons', () => { + const event = generateEvent({ + addons: { + vue: { + props: { + longValue: 'b'.repeat(5000), + }, + }, + }, + }); + + dataFilter.processEvent(event); + + expect(event.addons['vue']['props']['longValue']).toBe('b'.repeat(200) + '…'); + }); + + test('should trim long backtrace frame arguments', () => { + const event = generateEvent({ + backtrace: [ { + file: 'index.js', + line: 1, + arguments: [ 'c'.repeat(5000) ], + } ], + }); + + dataFilter.processEvent(event); + + expect(event.backtrace[0].arguments[0]).toBe('c'.repeat(200) + '…'); + }); + + test('should sanitize breadcrumbs message and data', () => { + const event = generateEvent({ + breadcrumbs: [ { + timestamp: 1701867896789, + message: 'd'.repeat(5000), + data: { + longValue: 'e'.repeat(5000), + }, + } ], + }); + + dataFilter.processEvent(event); + + expect(event.breadcrumbs[0].message).toBe('d'.repeat(200) + '…'); + expect(event.breadcrumbs[0].data['longValue']).toBe('e'.repeat(200) + '…'); + }); + + test('should replace circular references with a placeholder', () => { + const circular: Record = { + name: 'circular', + }; + + circular.self = circular; + + const event = generateEvent({ + context: circular as Json, + }); + + dataFilter.processEvent(event); + + expect(event.context['name']).toBe('circular'); + expect(event.context['self']).toBe(''); }); }); }); From 1443d747e8b94830b06fe0c317b00f8e5db2f02e Mon Sep 17 00:00:00 2001 From: Kuchizu Date: Tue, 14 Jul 2026 15:48:16 +0300 Subject: [PATCH 5/5] Keep sanitizer placeholder for context/addons wrapped in object --- workers/grouper/src/data-filter.ts | 7 ++++++- workers/grouper/tests/data-filter.test.ts | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/workers/grouper/src/data-filter.ts b/workers/grouper/src/data-filter.ts index bd89d175..6ae5cb90 100644 --- a/workers/grouper/src/data-filter.ts +++ b/workers/grouper/src/data-filter.ts @@ -173,7 +173,12 @@ export default class DataFilter { public sanitizeEvent(event: EventData): void { unsafeFields.forEach(field => { if (event[field] !== undefined) { - (event as unknown as Record)[field] = Sanitizer.sanitize(event[field]); + const sanitized = Sanitizer.sanitize(event[field]); + + (event as unknown as Record)[field] = typeof sanitized === 'string' + // eslint-disable-next-line @typescript-eslint/naming-convention + ? { __placeholder: sanitized } + : sanitized; } }); diff --git a/workers/grouper/tests/data-filter.test.ts b/workers/grouper/tests/data-filter.test.ts index d472244b..5ac57025 100644 --- a/workers/grouper/tests/data-filter.test.ts +++ b/workers/grouper/tests/data-filter.test.ts @@ -479,6 +479,26 @@ describe('GrouperWorker', () => { expect(event.breadcrumbs[0].data['longValue']).toBe('e'.repeat(200) + '…'); }); + test('should wrap string placeholder in an object when the whole context is replaced', () => { + const bigContext: Record = {}; + + for (let i = 0; i < 25; i++) { + bigContext[`key${i}`] = i; + } + + const event = generateEvent({ + context: bigContext, + }); + + dataFilter.processEvent(event); + + /** + * Raw string would be dropped by encodeUnsafeFields, so it must stay wrapped in an object + */ + // eslint-disable-next-line @typescript-eslint/naming-convention + expect(event.context).toEqual({ __placeholder: '' }); + }); + test('should replace circular references with a placeholder', () => { const circular: Record = { name: 'circular',