diff --git a/lib/utils/sanitizer.ts b/lib/utils/sanitizer.ts new file mode 100644 index 00000000..dd79617c --- /dev/null +++ b/lib/utils/sanitizer.ts @@ -0,0 +1,122 @@ +import { rightTrim } from './string'; + +/** + * Maximum string length before appending ellipsis + */ +const MAX_STRING_LENGTH = 200; + +/** + * Objects with more keys are represented as "" + */ +const MAX_OBJECT_KEYS_COUNT = 20; + +/** + * Maximum depth of sanitized objects + */ +const MAX_DEPTH = 5; + +/** + * Maximum length of sanitized arrays + */ +const MAX_ARRAY_LENGTH = 10; + +/** + * Checks that the value is a plain object + * + * @param target - value to check + */ +function isPlainObject(target: unknown): target is Record { + return Object.prototype.toString.call(target) === '[object Object]'; +} + +/** + * Values that can be tracked by WeakSet to detect circular references + */ +type ObjectLike = Record | unknown[]; + +/** + * Prepares event data for storing: trims long strings, slices long arrays, + * replaces too deep/big objects and circular references with placeholders. + */ +export class Sanitizer { + /** + * Apply sanitizing for array/object/primitives + * + * @param data - any value to sanitize + * @param depth - current depth of recursion + * @param seen - already visited objects + */ + public static sanitize(data: unknown, depth = 0, seen = new WeakSet()): unknown { + if (data !== null && typeof data === 'object') { + if (seen.has(data as ObjectLike)) { + return ''; + } + seen.add(data as ObjectLike); + } + + if (Array.isArray(data)) { + return Sanitizer.sanitizeArray(data, depth + 1, seen); + } + + if (isPlainObject(data)) { + return Sanitizer.sanitizeObject(data, depth + 1, seen); + } + + if (typeof data === 'string') { + return rightTrim(data, MAX_STRING_LENGTH); + } + + return data; + } + + /** + * Slices array to the maximum length and sanitizes each element + * + * @param arr - array to sanitize + * @param depth - current depth of recursion + * @param seen - already visited objects + */ + private static sanitizeArray(arr: unknown[], depth: number, seen: WeakSet): unknown[] { + const length = arr.length; + + if (length > MAX_ARRAY_LENGTH) { + arr = arr.slice(0, MAX_ARRAY_LENGTH); + arr.push(`<${length - MAX_ARRAY_LENGTH} more items...>`); + } + + return arr.map((item) => { + return Sanitizer.sanitize(item, depth, seen); + }); + } + + /** + * Sanitizes object values recursively + * + * @param data - object to sanitize + * @param depth - current depth of recursion + * @param seen - already visited objects + */ + private static sanitizeObject( + data: Record, + depth: number, + seen: WeakSet + ): Record | '' | '' { + if (depth > MAX_DEPTH) { + return ''; + } + + if (Object.keys(data).length > MAX_OBJECT_KEYS_COUNT) { + return ''; + } + + const result: Record = {}; + + for (const key in data) { + if (Object.prototype.hasOwnProperty.call(data, key)) { + result[key] = Sanitizer.sanitize(data[key], depth, seen); + } + } + + return result; + } +} diff --git a/workers/grouper/src/data-filter.ts b/workers/grouper/src/data-filter.ts index 2345b8e5..6ae5cb90 100644 --- a/workers/grouper/src/data-filter.ts +++ b/workers/grouper/src/data-filter.ts @@ -1,11 +1,18 @@ import type { EventAddons, EventData } from '@hawk.so/types'; import { unsafeFields } from '../../../lib/utils/unsafeFields'; +import { rightTrim } from '../../../lib/utils/string'; +import { Sanitizer } from '../../../lib/utils/sanitizer'; /** * Maximum depth for object traversal to prevent excessive memory allocations */ const MAX_TRAVERSAL_DEPTH = 20; +/** + * Maximum length for event title before appending ellipsis + */ +const MAX_TITLE_LENGTH = 400; + /** * Recursively iterate through object and call function on each key * @@ -135,6 +142,9 @@ export default class DataFilter { * @param event - event to process */ public processEvent(event: EventData): void { + this.trimEventTitle(event); + this.sanitizeEvent(event); + unsafeFields.forEach(field => { if (event[field]) { this.processField(event[field]); @@ -142,6 +152,53 @@ export default class DataFilter { }); } + /** + * Trim event title to the maximum allowed length. + * It mutates the original object. + * + * @param event - event to process + */ + public trimEventTitle(event: EventData): void { + if (typeof event.title === 'string') { + event.title = rightTrim(event.title, MAX_TITLE_LENGTH); + } + } + + /** + * Sanitize event fields that can contain long strings, deep objects or long arrays. + * It mutates the original object. + * + * @param event - event to process + */ + public sanitizeEvent(event: EventData): void { + unsafeFields.forEach(field => { + if (event[field] !== undefined) { + const sanitized = Sanitizer.sanitize(event[field]); + + (event as unknown as Record)[field] = typeof sanitized === 'string' + // eslint-disable-next-line @typescript-eslint/naming-convention + ? { __placeholder: sanitized } + : sanitized; + } + }); + + event.backtrace?.forEach(frame => { + if (frame.arguments !== undefined) { + frame.arguments = Sanitizer.sanitize(frame.arguments) as string[]; + } + }); + + event.breadcrumbs?.forEach(breadcrumb => { + if (typeof breadcrumb.message === 'string') { + breadcrumb.message = Sanitizer.sanitize(breadcrumb.message) as string; + } + + if (breadcrumb.data !== undefined) { + breadcrumb.data = Sanitizer.sanitize(breadcrumb.data) as typeof breadcrumb.data; + } + }); + } + /** * Recursively iterates object and applies filtering to its entries * diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 8203e8d3..46f77c98 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -59,7 +59,7 @@ const DB_DUPLICATE_KEY_ERROR = '11000'; const DAILY_METRICS_RETENTION_DAYS = 90; /** - * Maximum length for backtrace code line or title + * Maximum length for backtrace code line */ const MAX_CODE_LINE_LENGTH = 140; @@ -198,8 +198,6 @@ export default class GrouperWorker extends Worker { this.grouperMetrics.observePayloadSize(taskPayloadSize); this.memoryMonitor.logBeforeHandle(memoryBeforeHandle, handledTasksCount, taskPayloadSize, task.projectId); - this.logger.info(`[handle] project=${task.projectId} catcher=${task.catcherType} title="${task.payload.title}" payloadSize=${taskPayloadSize}b backtraceFrames=${task.payload.backtrace?.length ?? 0}`); - // FIX RELEASE TYPE // TODO: REMOVE AFTER 01.01.2026, after the most of the users update to new js catcher if (task.payload && task.payload.release !== undefined) { @@ -209,6 +207,11 @@ export default class GrouperWorker extends Worker { }; } + /** + * Filter event data before hashing so hash and stored event stay consistent. + */ + this.dataFilter.processEvent(task.payload); + let uniqueEventHash = await session.measureStep('hash', () => this.getUniqueEventHash(task)); let existedEvent: GroupedEventDBScheme; let repetitionId = null; @@ -219,11 +222,6 @@ export default class GrouperWorker extends Worker { * Trim source code lines to prevent memory leaks */ this.trimSourceCodeLines(task.payload); - - /** - * Filter sensitive information - */ - this.dataFilter.processEvent(task.payload); }); /** diff --git a/workers/grouper/tests/data-filter.test.ts b/workers/grouper/tests/data-filter.test.ts index 2f00dd68..5ac57025 100644 --- a/workers/grouper/tests/data-filter.test.ts +++ b/workers/grouper/tests/data-filter.test.ts @@ -14,16 +14,24 @@ jest.mock('amqplib'); * @param [options.context] - generated event context * @param [options.addons] - generated event addons */ -function generateEvent({ context, addons }: {context?: Json, addons?: EventAddons}): EventData { +function generateEvent({ context, addons, backtrace, breadcrumbs }: { + context?: Json, + addons?: EventAddons, + backtrace?: EventData['backtrace'], + breadcrumbs?: EventData['breadcrumbs'], +}): EventData { return { title: 'Event with sensitive data', - backtrace: [], + backtrace: backtrace ?? [], ...(context && { context, }), ...(addons && { addons, }), + ...(breadcrumbs && { + breadcrumbs, + }), }; } @@ -161,30 +169,52 @@ describe('GrouperWorker', () => { }); test('should filter additional sensitive keys (authorization, token, payment, dsn, ssn, etc.) in context', async () => { + /** + * Split the mock into two groups to keep objects under the sanitizer keys limit + */ + const entries = Object.entries(additionalSensitiveDataMock); + const half = Math.ceil(entries.length / 2); const event = generateEvent({ - context: additionalSensitiveDataMock, + context: { + group1: Object.fromEntries(entries.slice(0, half)), + group2: Object.fromEntries(entries.slice(half)), + }, }); dataFilter.processEvent(event); - Object.keys(additionalSensitiveDataMock).forEach((key) => { - expect(event.context[key]).toBe('[filtered]'); + entries.slice(0, half).forEach(([ key ]) => { + expect(event.context['group1'][key]).toBe('[filtered]'); + }); + entries.slice(half).forEach(([ key ]) => { + expect(event.context['group2'][key]).toBe('[filtered]'); }); }); test('should filter additional sensitive keys in addons', async () => { + /** + * Split the mock into two groups to keep objects under the sanitizer keys limit + */ + const entries = Object.entries(additionalSensitiveDataMock); + const half = Math.ceil(entries.length / 2); const event = generateEvent({ addons: { vue: { - props: additionalSensitiveDataMock, + props: { + group1: Object.fromEntries(entries.slice(0, half)), + group2: Object.fromEntries(entries.slice(half)), + }, }, }, }); dataFilter.processEvent(event); - Object.keys(additionalSensitiveDataMock).forEach((key) => { - expect(event.addons['vue']['props'][key]).toBe('[filtered]'); + entries.slice(0, half).forEach(([ key ]) => { + expect(event.addons['vue']['props']['group1'][key]).toBe('[filtered]'); + }); + entries.slice(half).forEach(([ key ]) => { + expect(event.addons['vue']['props']['group2'][key]).toBe('[filtered]'); }); }); @@ -328,9 +358,9 @@ describe('GrouperWorker', () => { expect(event.context['auth']).toBe('[filtered]'); }); - test('should handle deeply nested objects (>20 levels) without excessive memory allocations', () => { - // Create an object nested deeper than the cap (>20 levels) - let deeplyNested: any = { value: 'leaf', secret: 'should-be-filtered' }; + test('should replace too deep objects with a placeholder and keep filtering reachable levels', () => { + // Create an object nested deeper than the sanitizer depth cap + let deeplyNested: any = { value: 'leaf', secret: 'should-be-cut-off' }; for (let i = 0; i < 25; i++) { deeplyNested = { [`level${i}`]: deeplyNested, password: `sensitive${i}` }; @@ -343,23 +373,147 @@ describe('GrouperWorker', () => { // This should not throw or cause memory issues dataFilter.processEvent(event); - // Verify that filtering still works at various depths + // Filtering still works on the levels kept by the sanitizer expect(event.context['password']).toBe('[filtered]'); - // Navigate to a mid-level and check filtering - let current = event.context['level24'] as any; - for (let i = 24; i > 15; i--) { - expect(current['password']).toBe('[filtered]'); - current = current[`level${i - 1}`]; + const deepestKeptLevel = event.context['level24']['level23']['level22']['level21']; + + expect(deepestKeptLevel['password']).toBe('[filtered]'); + + // Everything deeper is replaced with a placeholder + expect(deepestKeptLevel['level20']).toBe(''); + }); + }); + + describe('Sanitizer', () => { + test('should trim long strings in context', () => { + const longString = 'a'.repeat(5000); + const event = generateEvent({ + context: { + longValue: longString, + }, + }); + + dataFilter.processEvent(event); + + expect(event.context['longValue']).toBe('a'.repeat(200) + '…'); + }); + + test('should replace objects with too many keys with a placeholder', () => { + const bigObject: Record = {}; + + for (let i = 0; i < 25; i++) { + bigObject[`key${i}`] = i; } - // At the leaf level, the secret should still be filtered - // (though path tracking may be capped, filtering should still work) - let leaf = event.context; - for (let i = 24; i >= 0; i--) { - leaf = leaf[`level${i}`] as any; + const event = generateEvent({ + context: { + bigObject, + }, + }); + + dataFilter.processEvent(event); + + expect(event.context['bigObject']).toBe(''); + }); + + test('should slice long arrays and add a placeholder', () => { + const event = generateEvent({ + context: { + longArray: new Array(15).fill('item') as unknown as Json, + }, + }); + + dataFilter.processEvent(event); + + const sanitizedArray = event.context['longArray']; + + expect(sanitizedArray).toHaveLength(11); // 10 items + placeholder + expect(sanitizedArray[10]).toBe('<5 more items...>'); + }); + + test('should sanitize addons', () => { + const event = generateEvent({ + addons: { + vue: { + props: { + longValue: 'b'.repeat(5000), + }, + }, + }, + }); + + dataFilter.processEvent(event); + + expect(event.addons['vue']['props']['longValue']).toBe('b'.repeat(200) + '…'); + }); + + test('should trim long backtrace frame arguments', () => { + const event = generateEvent({ + backtrace: [ { + file: 'index.js', + line: 1, + arguments: [ 'c'.repeat(5000) ], + } ], + }); + + dataFilter.processEvent(event); + + expect(event.backtrace[0].arguments[0]).toBe('c'.repeat(200) + '…'); + }); + + test('should sanitize breadcrumbs message and data', () => { + const event = generateEvent({ + breadcrumbs: [ { + timestamp: 1701867896789, + message: 'd'.repeat(5000), + data: { + longValue: 'e'.repeat(5000), + }, + } ], + }); + + dataFilter.processEvent(event); + + expect(event.breadcrumbs[0].message).toBe('d'.repeat(200) + '…'); + expect(event.breadcrumbs[0].data['longValue']).toBe('e'.repeat(200) + '…'); + }); + + test('should wrap string placeholder in an object when the whole context is replaced', () => { + const bigContext: Record = {}; + + for (let i = 0; i < 25; i++) { + bigContext[`key${i}`] = i; } - expect(leaf['secret']).toBe('[filtered]'); + + const event = generateEvent({ + context: bigContext, + }); + + dataFilter.processEvent(event); + + /** + * Raw string would be dropped by encodeUnsafeFields, so it must stay wrapped in an object + */ + // eslint-disable-next-line @typescript-eslint/naming-convention + expect(event.context).toEqual({ __placeholder: '' }); + }); + + test('should replace circular references with a placeholder', () => { + const circular: Record = { + name: 'circular', + }; + + circular.self = circular; + + const event = generateEvent({ + context: circular as Json, + }); + + dataFilter.processEvent(event); + + expect(event.context['name']).toBe('circular'); + expect(event.context['self']).toBe(''); }); }); }); diff --git a/workers/grouper/tests/index.test.ts b/workers/grouper/tests/index.test.ts index c75351a9..e1761bc5 100644 --- a/workers/grouper/tests/index.test.ts +++ b/workers/grouper/tests/index.test.ts @@ -173,6 +173,20 @@ describe('GrouperWorker', () => { expect(await eventsCollection.find().count()).toBe(1); }); + test('Should trim event title to the maximum length before saving', async () => { + const longTitle = 'A'.repeat(5000); + + await worker.handle(generateTask({ title: longTitle })); + + const savedEvent = await eventsCollection.findOne({}); + + /** + * 400 chars + ellipsis + */ + expect(savedEvent.payload.title.length).toBe(401); + expect(savedEvent.payload.title.endsWith('…')).toBe(true); + }); + test('Should increment total events count on each processing', async () => { await worker.handle(generateTask()); await worker.handle(generateTask());