From 1e5f103e73f5b8eadc43574a3d0266492c76e811 Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Tue, 21 Jan 2025 18:51:24 -0300 Subject: [PATCH 1/9] feat: telemetry service --- .../core/super-converter/SuperConverter.js | 17 ++ .../src/core/super-converter/Telemetry.js | 232 ++++++++++++++++++ .../v2/importer/docxImporter.js | 161 +++++++++--- 3 files changed, 377 insertions(+), 33 deletions(-) create mode 100644 packages/super-editor/src/core/super-converter/Telemetry.js diff --git a/packages/super-editor/src/core/super-converter/SuperConverter.js b/packages/super-editor/src/core/super-converter/SuperConverter.js index 08e4e75831..03f4aff8bf 100644 --- a/packages/super-editor/src/core/super-converter/SuperConverter.js +++ b/packages/super-editor/src/core/super-converter/SuperConverter.js @@ -3,6 +3,7 @@ import xmljs from 'xml-js'; import { DocxExporter, exportSchemaToJson } from './exporter'; import { createDocumentJson } from './v2/importer/docxImporter.js'; import { getArrayBufferFromUrl } from './helpers.js'; +import { Telemetry } from './Telemetry.js'; class SuperConverter { static allowedElements = Object.freeze({ @@ -95,6 +96,17 @@ class SuperConverter { this.tagsNotInSchema = ['w:body']; this.savedTagsToRestore = []; + // Initialize telemetry + this.telemetry = new Telemetry({ + enabled: params?.telemetry?.enabled ?? true, + dsn: params?.telemetry?.dsn, + endpoint: params?.telemetry?.endpoint + }); + + // Track unknown marks/elements for telemetry + this.unknownMarks = []; + this.unknownElements = []; + // Parse the initial XML, if provided if (this.docx.length || this.xml) this.parseFromXml(); } @@ -243,6 +255,11 @@ class SuperConverter { this.media = this.convertedXml.media; this.addedMedia = processedData; } + + destroy() { + // Clean up telemetry when converter is destroyed + return this.telemetry?.destroy(); + } } export { SuperConverter }; diff --git a/packages/super-editor/src/core/super-converter/Telemetry.js b/packages/super-editor/src/core/super-converter/Telemetry.js new file mode 100644 index 0000000000..2f6aba5bb1 --- /dev/null +++ b/packages/super-editor/src/core/super-converter/Telemetry.js @@ -0,0 +1,232 @@ +/** + * @typedef {Object} BaseEvent + * @property {string} id - Unique event identifier + * @property {string} timestamp - ISO timestamp of the event + * @property {string} sessionId - Current session identifier + */ + +/** + * @typedef {Object} UsageEvent + * @property {'usage'} type - Event type identifier + * @property {string} name - Name of the usage event + * @property {Object.} properties - Event properties + */ + +/** + * @typedef {Object} ParsingEvent + * @property {'parsing'} type - Event type identifier + * @property {'mark'|'element'} category - Category of the parsed item + * @property {string} name - Name of the parsed item + * @property {string} path - Document path where item was found + * @property {Object.} [metadata] - Additional context + */ + +/** @typedef {(UsageEvent & BaseEvent) | (ParsingEvent & BaseEvent)} TelemetryEvent */ + +/** + * @typedef {Object} TelemetryConfig + * @property {string} [dsn] - Data Source Name for telemetry service + * @property {string} [endpoint] - Optional override for telemetry endpoint + * @property {boolean} [enabled=true] - Whether telemetry is enabled + */ + +class Telemetry { + /** @type {boolean} */ + #enabled; + + /** @type {string} */ + #endpoint; + + /** @type {string} */ + #projectId; + + /** @type {string} */ + #token; + + /** @type {string} */ + #sessionId; + + /** @type {TelemetryEvent[]} */ + #events = []; + + /** @type {number|undefined} */ + #flushInterval; + + /** @type {number} */ + static BATCH_SIZE = 50; + + /** @type {number} */ + static FLUSH_INTERVAL = 30000; // 30 seconds + + /** @type {string} */ + static COMMUNITY_DSN = 'https://public@telemetry.superdoc.dev/community'; + + /** + * Initialize telemetry service + * @param {TelemetryConfig} config + */ + constructor(config = {}) { + this.#enabled = config.enabled ?? true; + + try { + const dsn = config.dsn ?? Telemetry.COMMUNITY_DSN; + const { projectId, token, endpoint } = this.#parseDsn(dsn); + this.#projectId = projectId; + this.#token = token; + this.#endpoint = config.endpoint ?? endpoint; + } catch (error) { + console.warn('Invalid telemetry configuration, disabling:', error); + this.#enabled = false; + return; + } + + this.#sessionId = this.#generateId(); + + if (this.#enabled) { + this.#startPeriodicFlush(); + } + } + + /** + * Track feature usage + * @param {string} name - Name of the feature/event + * @param {Object.} [properties] - Additional properties + */ + trackUsage(name, properties = {}) { + if (!this.#enabled) return; + + /** @type {UsageEvent & BaseEvent} */ + const event = { + id: this.#generateId(), + type: 'usage', + timestamp: new Date().toISOString(), + sessionId: this.#sessionId, + name, + properties + }; + + this.#queueEvent(event); + } + + /** + * Track parsing events + * @param {'mark'|'element'} category - Category of parsed item + * @param {string} name - Name of the item + * @param {string} path - Document path where item was found + * @param {Object.} [metadata] - Additional context + */ + trackParsing(category, name, path, metadata) { + if (!this.#enabled) return; + + /** @type {ParsingEvent & BaseEvent} */ + const event = { + id: this.#generateId(), + type: 'parsing', + timestamp: new Date().toISOString(), + sessionId: this.#sessionId, + category, + name, + path, + ...(metadata && { metadata }) + }; + + this.#queueEvent(event); + } + + /** + * Queue event for sending + * @param {TelemetryEvent} event + * @private + */ + #queueEvent(event) { + this.#events.push(event); + + if (this.#events.length >= Telemetry.BATCH_SIZE) { + this.flush(); + } + } + + /** + * Flush queued events to server + * @returns {Promise} + */ + async flush() { + if (!this.#enabled || this.#events.length === 0) return; + + const eventsToSend = [...this.#events]; + this.#events = []; + + try { + const response = await fetch(this.#endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Project-ID': this.#projectId, + 'X-Token': this.#token + }, + body: JSON.stringify(eventsToSend) + }); + + if (!response.ok) { + throw new Error(`Upload failed: ${response.statusText}`); + } + } catch (error) { + console.error('Failed to upload telemetry:', error); + // Add events back to queue + this.#events = [...eventsToSend, ...this.#events]; + } + } + + /** + * Start periodic flush interval + * @private + */ + #startPeriodicFlush() { + this.#flushInterval = setInterval(() => { + if (this.#events.length > 0) { + this.flush(); + } + }, Telemetry.FLUSH_INTERVAL); + } + + /** + * Parse DSN string into config + * @param {string} dsn + * @returns {{ projectId: string, token: string, endpoint: string }} + * @private + */ + #parseDsn(dsn) { + try { + const url = new URL(dsn); + return { + token: url.username, + projectId: url.pathname.split('/')[1], + endpoint: `${url.protocol}//${url.host}/v1/collect` + }; + } catch (error) { + throw new Error(`Invalid DSN: ${error.message}`); + } + } + + /** + * Generate unique identifier + * @returns {string} + * @private + */ + #generateId() { + return `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; + } + + /** + * Clean up telemetry service + * @returns {Promise} + */ + destroy() { + if (this.#flushInterval) { + clearInterval(this.#flushInterval); + } + return this.flush(); + } +} + +export { Telemetry }; \ No newline at end of file diff --git a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js index 743ca5718e..8731723d46 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js @@ -44,7 +44,7 @@ export const createDocumentJson = (docx, converter) => { const ignoreNodes = ['w:sectPr']; const content = node.elements?.filter((n) => !ignoreNodes.includes(n.name)) ?? []; - const parsedContent = nodeListHandler.handler(content, docx, false); + const parsedContent = nodeListHandler.handler(content, docx, false, converter); const result = { type: 'doc', content: parsedContent, @@ -91,48 +91,143 @@ export const defaultNodeListHandler = () => { */ const createNodeListHandler = (nodeHandlers) => { /** - * @param {XmlNode[]} elements - * @param {ParsedDocx} docx - * @param {boolean} insideTrackChange - * @param {string} filename - * @return {{type: string, content: *, attrs: {attributes}}[]} + * Gets safe element context even if index is out of bounds + * @param {Array} elements Array of elements + * @param {number} index Index to check + * @returns {Object} Safe context object */ - const nodeListHandlerFn = (elements, docx, insideTrackChange, filename) => { + const getSafeElementContext = (elements, index) => { + if (!elements || index < 0 || index >= elements.length) { + return { + elementIndex: index, + error: 'index_out_of_bounds', + arrayLength: elements?.length + }; + } + + const element = elements[index]; + return { + elementIndex: index, + elementName: element?.name, + elementAttributes: element?.attributes, + hasElements: !!element?.elements, + elementCount: element?.elements?.length + }; + }; + + const nodeListHandlerFn = (elements, docx, insideTrackChange, converter, filename) => { if (!elements || !elements.length) return []; + const processedElements = []; + const unhandledNodes = []; - for (let index = 0; index < elements.length; index++) { - const { nodes, consumed } = nodeHandlers.reduce( - (res, handler) => { - if (res.consumed > 0) return res; + try { + for (let index = 0; index < elements.length; index++) { + try { const nodesToHandle = elements.slice(index); - if (!nodesToHandle || nodesToHandle.length === 0) return res; - const result = handler.handler( - nodesToHandle, - docx, - { handler: nodeListHandlerFn, handlerEntities: nodeHandlers }, - insideTrackChange, - filename, + if (!nodesToHandle || nodesToHandle.length === 0) { + converter?.telemetry?.trackParsing('node', 'empty_slice', `/word/${filename || 'document.xml'}`, { + index, + totalElements: elements.length + }); + continue; + } + + const { nodes, consumed } = nodeHandlers.reduce( + (res, handler) => { + if (res.consumed > 0) return res; + + return handler.handler( + nodesToHandle, + docx, + { handler: nodeListHandlerFn, handlerEntities: nodeHandlers }, + insideTrackChange, + converter, + filename + ); + }, + { nodes: [], consumed: 0 } ); - return result; - }, - { nodes: [], consumed: 0 }, - ); - index += consumed - 1; - if (consumed === 0) { - console.warn("We have a node that we can't handle!", elements[index]); - } - for (let node of nodes) { - if (node?.type) { - const ignore = ['runProperties']; - // Ignore empty text nodes - if (node.type === 'text' && Array.isArray(node.content) && !node.content.length) continue; - if (!ignore.includes(node.type)) processedElements.push(node); + // Bounds check before incrementing index + if (consumed > 0) { + index += consumed - 1; + if (index < 0) { + converter?.telemetry?.trackParsing('node', 'invalid_index', `/word/${filename || 'document.xml'}`, { + originalIndex: index - (consumed - 1), + consumed, + resultingIndex: index + }); + index = 0; // Reset to safe value + } + } + + // Track unhandled nodes with safe context + if (consumed === 0) { + const context = getSafeElementContext(elements, index); + unhandledNodes.push({ + name: context.elementName, + attributes: context.elementAttributes + }); + + converter?.telemetry?.trackParsing('node', 'unhandled', `/word/${filename || 'document.xml'}`, { + context + }); + continue; + } + + // Process valid nodes + for (let node of (nodes || [])) { + if (node?.type) { + const ignore = ['runProperties']; + + if (node.type === 'text' && Array.isArray(node.content) && !node.content.length) { + converter?.telemetry?.trackParsing('node', 'empty_text', `/word/${filename || 'document.xml'}`, { + context: node.attrs + }); + continue; + } + + if (!ignore.includes(node.type)) { + processedElements.push(node); + } + } + } + } catch (error) { + // Track individual element processing errors with safe context + converter?.telemetry?.trackParsing('element', 'processing_error', `/word/${filename || 'document.xml'}`, { + error: { + message: error.message, + name: error.name, + stack: error.stack + }, + context: getSafeElementContext(elements, index) + }); + + continue; } } + + return processedElements; + + } catch (error) { + // Track catastrophic errors in the node list handler + converter?.telemetry?.trackParsing('handler', 'nodeListHandler', `/word/${filename || 'document.xml'}`, { + status: 'error', + error: { + message: error.message, + name: error.name, + stack: error.stack + }, + context: { + totalElements: elements?.length || 0, + processedCount: processedElements.length, + unhandledCount: unhandledNodes.length + } + }); + + throw error; } - return processedElements; }; return nodeListHandlerFn; From d8848183dd0ec6b05901f91f0bf8161bfaf1a8bc Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Wed, 22 Jan 2025 11:40:34 -0300 Subject: [PATCH 2/9] feat: add doc info --- .../src/core/super-converter/Telemetry.js | 75 ++++++++++++++----- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/packages/super-editor/src/core/super-converter/Telemetry.js b/packages/super-editor/src/core/super-converter/Telemetry.js index 2f6aba5bb1..ed7b42ba39 100644 --- a/packages/super-editor/src/core/super-converter/Telemetry.js +++ b/packages/super-editor/src/core/super-converter/Telemetry.js @@ -3,6 +3,12 @@ * @property {string} id - Unique event identifier * @property {string} timestamp - ISO timestamp of the event * @property {string} sessionId - Current session identifier + * @property {Object} document - Document information + * @property {string} [document.id] - Reference ID + * @property {string} [document.type] - Document type + * @property {string} [document.internalId] - Internal document ID + * @property {string} [document.hash] - Document MD5 hash + * @property {string} [document.lastModified] - Last modified timestamp */ /** @@ -33,22 +39,22 @@ class Telemetry { /** @type {boolean} */ #enabled; - + /** @type {string} */ #endpoint; - + /** @type {string} */ #projectId; - + /** @type {string} */ #token; - + /** @type {string} */ #sessionId; - + /** @type {TelemetryEvent[]} */ #events = []; - + /** @type {number|undefined} */ #flushInterval; @@ -67,7 +73,7 @@ class Telemetry { */ constructor(config = {}) { this.#enabled = config.enabled ?? true; - + try { const dsn = config.dsn ?? Telemetry.COMMUNITY_DSN; const { projectId, token, endpoint } = this.#parseDsn(dsn); @@ -81,7 +87,7 @@ class Telemetry { } this.#sessionId = this.#generateId(); - + if (this.#enabled) { this.#startPeriodicFlush(); } @@ -90,9 +96,10 @@ class Telemetry { /** * Track feature usage * @param {string} name - Name of the feature/event + * @param {Object} document - Document information * @param {Object.} [properties] - Additional properties */ - trackUsage(name, properties = {}) { + trackUsage(name, document = {}, properties = {}) { if (!this.#enabled) return; /** @type {UsageEvent & BaseEvent} */ @@ -102,7 +109,8 @@ class Telemetry { timestamp: new Date().toISOString(), sessionId: this.#sessionId, name, - properties + document, + properties, }; this.#queueEvent(event); @@ -113,9 +121,10 @@ class Telemetry { * @param {'mark'|'element'} category - Category of parsed item * @param {string} name - Name of the item * @param {string} path - Document path where item was found + * @param {Object} document - Document information * @param {Object.} [metadata] - Additional context */ - trackParsing(category, name, path, metadata) { + trackParsing(category, name, path, document = {}, metadata) { if (!this.#enabled) return; /** @type {ParsingEvent & BaseEvent} */ @@ -127,12 +136,44 @@ class Telemetry { category, name, path, - ...(metadata && { metadata }) + document, + ...(metadata && { metadata }), }; this.#queueEvent(event); } + /** + * Process document metadata + * @param {File} file - Document file + * @param {Object} options - Additional metadata options + * @returns {Promise} Document metadata + */ + async processDocument(file, options = {}) { + const hash = await this.#generateMD5Hash(file); + + return { + id: options.id, + type: options.type || file.type, + internalId: options.internalId, + hash, + lastModified: file.lastModified ? new Date(file.lastModified).toISOString() : null, + }; + } + + /** + * Generate MD5 hash for a file + * @param {File} file - File to hash + * @returns {Promise} MD5 hash + * @private + */ + async #generateMD5Hash(file) { + const buffer = await file.arrayBuffer(); + const hashBuffer = await crypto.subtle.digest('MD5', buffer); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + return hashArray.map((b) => b.toString(16).padStart(2, '0')).join(''); + } + /** * Queue event for sending * @param {TelemetryEvent} event @@ -140,7 +181,7 @@ class Telemetry { */ #queueEvent(event) { this.#events.push(event); - + if (this.#events.length >= Telemetry.BATCH_SIZE) { this.flush(); } @@ -162,9 +203,9 @@ class Telemetry { headers: { 'Content-Type': 'application/json', 'X-Project-ID': this.#projectId, - 'X-Token': this.#token + 'X-Token': this.#token, }, - body: JSON.stringify(eventsToSend) + body: JSON.stringify(eventsToSend), }); if (!response.ok) { @@ -201,7 +242,7 @@ class Telemetry { return { token: url.username, projectId: url.pathname.split('/')[1], - endpoint: `${url.protocol}//${url.host}/v1/collect` + endpoint: `${url.protocol}//${url.host}/v1/collect`, }; } catch (error) { throw new Error(`Invalid DSN: ${error.message}`); @@ -229,4 +270,4 @@ class Telemetry { } } -export { Telemetry }; \ No newline at end of file +export { Telemetry }; From 548aeed7bdf735f77aaead94f0ab5593a2b250fb Mon Sep 17 00:00:00 2001 From: Vladyslava Andrushchenko Date: Wed, 22 Jan 2025 21:46:20 +0200 Subject: [PATCH 3/9] HAR-8546 Telemetry Service Updated --- packages/super-editor/src/core/Editor.js | 33 ++++++- .../core/{super-converter => }/Telemetry.js | 86 ++++++++++++++----- .../core/super-converter/SuperConverter.js | 11 +-- .../v2/importer/docxImporter.js | 51 ++++++----- .../v2/importer/listImporter.js | 4 +- .../v2/importer/runNodeImporter.js | 2 +- .../v2/importer/standardNodeImporter.js | 16 +++- .../dev/components/DeveloperPlayground.vue | 3 + packages/superdoc/src/SuperDoc.vue | 1 + packages/superdoc/src/core/SuperDoc.js | 3 + .../src/dev/components/SuperdocDev.vue | 3 + 11 files changed, 157 insertions(+), 56 deletions(-) rename packages/super-editor/src/core/{super-converter => }/Telemetry.js (75%) diff --git a/packages/super-editor/src/core/Editor.js b/packages/super-editor/src/core/Editor.js index 1ffd5de2e9..cdaa0b6e08 100644 --- a/packages/super-editor/src/core/Editor.js +++ b/packages/super-editor/src/core/Editor.js @@ -15,6 +15,7 @@ import { trackedTransaction } from '@extensions/track-changes/trackChangesHelper import { TrackChangesBasePluginKey } from '@extensions/track-changes/plugins/index.js'; import { initPaginationData, PaginationPluginKey } from '@extensions/pagination/pagination-helpers'; import DocxZipper from '@core/DocxZipper.js'; +import { Telemetry } from '@core/Telemetry.js'; /** * Editor main class. @@ -81,6 +82,11 @@ export class Editor extends EventEmitter { onCollaborationReady: () => null, // async (file) => url; handleImageUpload: null, + + + // telemetry + telemetry: null, + telemetryService: null, }; constructor(options) { @@ -104,6 +110,7 @@ export class Editor extends EventEmitter { } #init(options) { + this.#initTelemetry(); this.#createExtensionService(); this.#createCommandService(); this.#createSchema(); @@ -438,6 +445,22 @@ export class Editor extends EventEmitter { docx: this.options.content, media: this.options.mediaFiles, debug: true, + telemetry: this.telemetryService, + }); + } + } + + /** + * Initialize telemetry service. + */ + #initTelemetry() { + if (this.options.telemetry?.enabled) { + this.telemetryService = new Telemetry({ + enabled: this.options.telemetry.enabled ?? true, + dsn: this.options.telemetry?.dsn, + endpoint: this.options.telemetry.endpoint, + fileSource: this.options.fileSource, + documentId: this.options.documentId, }); } } @@ -495,7 +518,7 @@ export class Editor extends EventEmitter { try { const { mode, fragment, isHeadless, content, loadFromSchema } = this.options; - + if (mode === 'docx') { doc = createDocument(this.converter, this.schema); @@ -838,6 +861,11 @@ export class Editor extends EventEmitter { fonts: this.options.fonts, isHeadless: this.options.isHeadless, }); + + this.telemetryService?.trackUsage('document_export', { + documentType: 'docx', + timestamp: new Date().toISOString() + }); return result; } @@ -860,5 +888,8 @@ export class Editor extends EventEmitter { if (this.view) this.view.destroy(); this.#endCollaboration(); this.removeAllListeners(); + + // Clean up telemetry when editor is destroyed + this.telemetry?.destroy(); } } diff --git a/packages/super-editor/src/core/super-converter/Telemetry.js b/packages/super-editor/src/core/Telemetry.js similarity index 75% rename from packages/super-editor/src/core/super-converter/Telemetry.js rename to packages/super-editor/src/core/Telemetry.js index ed7b42ba39..1c183d848f 100644 --- a/packages/super-editor/src/core/super-converter/Telemetry.js +++ b/packages/super-editor/src/core/Telemetry.js @@ -34,6 +34,8 @@ * @property {string} [dsn] - Data Source Name for telemetry service * @property {string} [endpoint] - Optional override for telemetry endpoint * @property {boolean} [enabled=true] - Whether telemetry is enabled + * @property {File} [fileSource] - current editor file + * @property {string} documentId - Reference ID */ class Telemetry { @@ -49,6 +51,9 @@ class Telemetry { /** @type {string} */ #token; + /** @type {object} */ + #document; + /** @type {string} */ #sessionId; @@ -62,7 +67,7 @@ class Telemetry { static BATCH_SIZE = 50; /** @type {number} */ - static FLUSH_INTERVAL = 30000; // 30 seconds + static FLUSH_INTERVAL = 5000; // 30 seconds /** @type {string} */ static COMMUNITY_DSN = 'https://public@telemetry.superdoc.dev/community'; @@ -76,13 +81,10 @@ class Telemetry { try { const dsn = config.dsn ?? Telemetry.COMMUNITY_DSN; - const { projectId, token, endpoint } = this.#parseDsn(dsn); - this.#projectId = projectId; - this.#token = token; - this.#endpoint = config.endpoint ?? endpoint; + this.initTelemetry(config, dsn); } catch (error) { - console.warn('Invalid telemetry configuration, disabling:', error); - this.#enabled = false; + console.warn('Invalid telemetry configuration, enabling community version:', error); + if (config.dsn) this.initTelemetry(config, Telemetry.COMMUNITY_DSN); return; } @@ -93,13 +95,41 @@ class Telemetry { } } + /** + * Init variables + * @param {TelemetryConfig} config + * @param {string} dsn - Data Source Name for telemetry service + */ + async initTelemetry(config, dsn) { + const { projectId, token, endpoint } = this.#parseDsn(dsn); + this.#projectId = projectId; + this.#token = token; + this.#endpoint = config.endpoint ?? endpoint; + this.#document = await this.processDocument(config.fileSource, { + id: config.documentId, + }); + } + + + getSourceData() { + return { + userAgent: window.navigator.userAgent, + url: window.location.href, + host: window.location.host, + referrer: document.referrer, + screen: { + width: window.screen.width, + height: window.screen.height, + }, + }; + } + /** * Track feature usage * @param {string} name - Name of the feature/event - * @param {Object} document - Document information * @param {Object.} [properties] - Additional properties */ - trackUsage(name, document = {}, properties = {}) { + trackUsage(name, properties = {}) { if (!this.#enabled) return; /** @type {UsageEvent & BaseEvent} */ @@ -108,8 +138,9 @@ class Telemetry { type: 'usage', timestamp: new Date().toISOString(), sessionId: this.#sessionId, + source: this.getSourceData(), name, - document, + document: this.#document, properties, }; @@ -121,10 +152,9 @@ class Telemetry { * @param {'mark'|'element'} category - Category of parsed item * @param {string} name - Name of the item * @param {string} path - Document path where item was found - * @param {Object} document - Document information * @param {Object.} [metadata] - Additional context */ - trackParsing(category, name, path, document = {}, metadata) { + trackParsing(category, name, path, metadata) { if (!this.#enabled) return; /** @type {ParsingEvent & BaseEvent} */ @@ -133,10 +163,11 @@ class Telemetry { type: 'parsing', timestamp: new Date().toISOString(), sessionId: this.#sessionId, + source: this.getSourceData(), category, name, path, - document, + document: this.#document, ...(metadata && { metadata }), }; @@ -150,15 +181,24 @@ class Telemetry { * @returns {Promise} Document metadata */ async processDocument(file, options = {}) { - const hash = await this.#generateMD5Hash(file); - - return { - id: options.id, - type: options.type || file.type, - internalId: options.internalId, - hash, - lastModified: file.lastModified ? new Date(file.lastModified).toISOString() : null, - }; + try { + const hash = await this.#generateMD5Hash(file); + + return { + id: options.id, + type: options.type || file.type, + internalId: options.internalId, + hash, + lastModified: file.lastModified ? new Date(file.lastModified).toISOString() : null, + }; + } catch (error) { + return { + id: options.id, + type: options.type || file.type, + internalId: options.internalId, + lastModified: file.lastModified ? new Date(file.lastModified).toISOString() : null, + } + } } /** @@ -169,7 +209,7 @@ class Telemetry { */ async #generateMD5Hash(file) { const buffer = await file.arrayBuffer(); - const hashBuffer = await crypto.subtle.digest('MD5', buffer); + const hashBuffer = await crypto.subtle.digest('SHA-256', buffer); const hashArray = Array.from(new Uint8Array(hashBuffer)); return hashArray.map((b) => b.toString(16).padStart(2, '0')).join(''); } diff --git a/packages/super-editor/src/core/super-converter/SuperConverter.js b/packages/super-editor/src/core/super-converter/SuperConverter.js index 03f4aff8bf..8376db4141 100644 --- a/packages/super-editor/src/core/super-converter/SuperConverter.js +++ b/packages/super-editor/src/core/super-converter/SuperConverter.js @@ -3,7 +3,6 @@ import xmljs from 'xml-js'; import { DocxExporter, exportSchemaToJson } from './exporter'; import { createDocumentJson } from './v2/importer/docxImporter.js'; import { getArrayBufferFromUrl } from './helpers.js'; -import { Telemetry } from './Telemetry.js'; class SuperConverter { static allowedElements = Object.freeze({ @@ -97,15 +96,10 @@ class SuperConverter { this.savedTagsToRestore = []; // Initialize telemetry - this.telemetry = new Telemetry({ - enabled: params?.telemetry?.enabled ?? true, - dsn: params?.telemetry?.dsn, - endpoint: params?.telemetry?.endpoint - }); + this.telemetry = params?.telemetry || null; - // Track unknown marks/elements for telemetry + // ToDo do we still need to track unknown marks for telemetry this.unknownMarks = []; - this.unknownElements = []; // Parse the initial XML, if provided if (this.docx.length || this.xml) this.parseFromXml(); @@ -188,6 +182,7 @@ class SuperConverter { getSchema() { const result = createDocumentJson({...this.convertedXml, media: this.media }, this ); + if (result) { this.savedTagsToRestore.push({ ...result.savedTagsToRestore }); this.pageStyles = result.pageStyles; diff --git a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js index 8731723d46..e930a48f5a 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js @@ -52,6 +52,15 @@ export const createDocumentJson = (docx, converter) => { attributes: json.elements[0].attributes, }, }; + + // Not empty document + if (result.content.length > 1) { + converter.telemetry?.trackUsage('document_import', { + documentType: 'docx', + timestamp: new Date().toISOString() + }); + } + return { pmDoc: result, savedTagsToRestore: node, @@ -120,7 +129,7 @@ const createNodeListHandler = (nodeHandlers) => { const processedElements = []; const unhandledNodes = []; - + try { for (let index = 0; index < elements.length; index++) { try { @@ -133,7 +142,7 @@ const createNodeListHandler = (nodeHandlers) => { continue; } - const { nodes, consumed } = nodeHandlers.reduce( + const { nodes, consumed, unhandled } = nodeHandlers.reduce( (res, handler) => { if (res.consumed > 0) return res; @@ -149,6 +158,23 @@ const createNodeListHandler = (nodeHandlers) => { { nodes: [], consumed: 0 } ); + // Track unhandled nodes with safe context + if (unhandled) { + const context = getSafeElementContext(elements, index); + if (!context.elementName) continue; + + unhandledNodes.push({ + name: context.elementName, + attributes: context.elementAttributes + }); + + converter?.telemetry?.trackParsing('node', 'unhandled', `/word/${filename || 'document.xml'}`, { + context + }); + + continue; + } + // Bounds check before incrementing index if (consumed > 0) { index += consumed - 1; @@ -161,27 +187,14 @@ const createNodeListHandler = (nodeHandlers) => { index = 0; // Reset to safe value } } - - // Track unhandled nodes with safe context - if (consumed === 0) { - const context = getSafeElementContext(elements, index); - unhandledNodes.push({ - name: context.elementName, - attributes: context.elementAttributes - }); - - converter?.telemetry?.trackParsing('node', 'unhandled', `/word/${filename || 'document.xml'}`, { - context - }); - continue; - } - + // Process valid nodes for (let node of (nodes || [])) { if (node?.type) { const ignore = ['runProperties']; if (node.type === 'text' && Array.isArray(node.content) && !node.content.length) { + converter?.telemetry?.trackParsing('node', 'empty_text', `/word/${filename || 'document.xml'}`, { context: node.attrs }); @@ -203,8 +216,6 @@ const createNodeListHandler = (nodeHandlers) => { }, context: getSafeElementContext(elements, index) }); - - continue; } } @@ -317,4 +328,4 @@ function getHeaderFooter(el, elementType, docx, converter) { storage[rId] = { type: 'doc', content: [...schema] }; storageIds[sectionType] = rId; -}; \ No newline at end of file +}; diff --git a/packages/super-editor/src/core/super-converter/v2/importer/listImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/listImporter.js index 252266d8b7..f53e7bf0ef 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/listImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/listImporter.js @@ -372,7 +372,7 @@ const getListLevelDefinitionTag = (numId, level, pStyleId, docx) => { } const start = currentLevel?.elements?.find((style) => style.name === 'w:start')?.attributes['w:val']; - const numFmt = currentLevel?.elements?.find((style) => style.name === 'w:numFmt').attributes['w:val']; + const numFmt = currentLevel?.elements?.find((style) => style.name === 'w:numFmt')?.attributes['w:val']; const lvlText = currentLevel?.elements?.find((style) => style.name === 'w:lvlText').attributes['w:val']; const lvlJc = currentLevel?.elements?.find((style) => style.name === 'w:lvlJc').attributes['w:val']; const pPr = currentLevel?.elements?.find((style) => style.name === 'w:pPr'); @@ -425,7 +425,7 @@ export function getNodeNumberingDefinition(attributes, level, docx) { // Get style for this list level let listType; - if (unorderedListTypes.includes(listTypeDef.toLowerCase())) listType = 'bulletList'; + if (unorderedListTypes.includes(listTypeDef?.toLowerCase())) listType = 'bulletList'; else if (orderedListTypes.includes(listTypeDef)) listType = 'orderedList'; else { throw new Error(`Unknown list type found during import: ${listTypeDef}`); diff --git a/packages/super-editor/src/core/super-converter/v2/importer/runNodeImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/runNodeImporter.js index 76409a748b..9804e80e66 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/runNodeImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/runNodeImporter.js @@ -8,7 +8,7 @@ const handleRunNode = (nodes, docx, nodeListHandler, insideTrackChange = false, if (nodes.length === 0 || nodes[0].name !== 'w:r') { return { nodes: [], consumed: 0 }; } - + const node = nodes[0]; let processedRun = nodeListHandler.handler(node.elements, docx, insideTrackChange, filename)?.filter((n) => n) || []; const hasRunProperties = node.elements.some((el) => el.name === 'w:rPr'); diff --git a/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js index 8c7ea5c5ae..eb95d00329 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js @@ -11,10 +11,24 @@ export const handleStandardNode = (nodes, docx, nodeListHandler, insideTrackChan // Parse properties const { name, type } = node; const { attributes, elements, marks = [] } = parseProperties(node, docx); + + if (!getElementName(node)) { + return { + nodes: [{ + type: name, + content: elements, + attrs: { ...attributes }, + marks, + }], + consumed: 0, + unhandled: true, + }; + } // Iterate through the children and build the schemaNode content + // Skip run properties since they are formatting only elements const content = []; - if (elements && elements.length) { + if (elements && elements.length && name !== 'w:rPr') { const updatedElements = elements.map((el) => { if (!el.marks) el.marks = []; el.marks.push(...marks); diff --git a/packages/super-editor/src/dev/components/DeveloperPlayground.vue b/packages/super-editor/src/dev/components/DeveloperPlayground.vue index 95b058d6a1..38ea28a510 100644 --- a/packages/super-editor/src/dev/components/DeveloperPlayground.vue +++ b/packages/super-editor/src/dev/components/DeveloperPlayground.vue @@ -65,6 +65,9 @@ const editorOptions = computed(() => { suppressSkeletonLoader: true, users: [], // For comment @-mentions, only users that have access to the document pagination: true, + telemetry: { + enabled: true + } } }); diff --git a/packages/superdoc/src/SuperDoc.vue b/packages/superdoc/src/SuperDoc.vue index 36c3cf7b6a..2b9d2c8880 100644 --- a/packages/superdoc/src/SuperDoc.vue +++ b/packages/superdoc/src/SuperDoc.vue @@ -303,6 +303,7 @@ const editorOptions = (doc) => { collaborationProvider: doc.provider || null, isNewFile: doc.isNewFile || false, handleImageUpload: proxy.$superdoc.config.handleImageUpload, + telemetry: proxy.$superdoc.telemetry, }; return options; diff --git a/packages/superdoc/src/core/SuperDoc.js b/packages/superdoc/src/core/SuperDoc.js index 9334cf7a64..2f9ab60bfc 100644 --- a/packages/superdoc/src/core/SuperDoc.js +++ b/packages/superdoc/src/core/SuperDoc.js @@ -56,6 +56,9 @@ export class SuperDoc extends EventEmitter { isDev: false, + // telemetry config + telemetry: null, + // Events onEditorBeforeCreate: () => null, onEditorCreate: () => null, diff --git a/packages/superdoc/src/dev/components/SuperdocDev.vue b/packages/superdoc/src/dev/components/SuperdocDev.vue index 655ac49516..89da943674 100644 --- a/packages/superdoc/src/dev/components/SuperdocDev.vue +++ b/packages/superdoc/src/dev/components/SuperdocDev.vue @@ -64,6 +64,9 @@ const init = async () => { // url: 'ws://localhost:3050/docs/superdoc-id', // } }, + telemetry: { + enabled: true, + }, onEditorCreate, onContentError, // handleImageUpload: async (file) => url, From e8fdaed16fd6edd82033065e27dfc3d0f99ba9ca Mon Sep 17 00:00:00 2001 From: Vladyslava Andrushchenko Date: Thu, 23 Jan 2025 15:23:45 +0200 Subject: [PATCH 4/9] collect internalId and error details --- packages/super-editor/src/core/Editor.js | 9 ++- packages/super-editor/src/core/Telemetry.js | 71 +++++++++++-------- .../src/core/helpers/ErrorWithDetails.js | 21 ++++++ .../core/super-converter/SuperConverter.js | 12 +++- .../v2/importer/docxImporter.js | 26 +++++-- .../v2/importer/listImporter.js | 6 +- 6 files changed, 103 insertions(+), 42 deletions(-) create mode 100644 packages/super-editor/src/core/helpers/ErrorWithDetails.js diff --git a/packages/super-editor/src/core/Editor.js b/packages/super-editor/src/core/Editor.js index cdaa0b6e08..18cae61141 100644 --- a/packages/super-editor/src/core/Editor.js +++ b/packages/super-editor/src/core/Editor.js @@ -453,7 +453,7 @@ export class Editor extends EventEmitter { /** * Initialize telemetry service. */ - #initTelemetry() { + async #initTelemetry() { if (this.options.telemetry?.enabled) { this.telemetryService = new Telemetry({ enabled: this.options.telemetry.enabled ?? true, @@ -580,6 +580,13 @@ export class Editor extends EventEmitter { }); } + /** + * Get document internal ID(w15:docId) from settings.xml + */ + initDocInternalId() { + this.options.documentInternalId = this.converter.getDocumentInternalId(); + } + /** * Initialize default styles for the editor container and ProseMirror. * Get page size and margins from the converter. diff --git a/packages/super-editor/src/core/Telemetry.js b/packages/super-editor/src/core/Telemetry.js index 1c183d848f..508df91f57 100644 --- a/packages/super-editor/src/core/Telemetry.js +++ b/packages/super-editor/src/core/Telemetry.js @@ -50,9 +50,9 @@ class Telemetry { /** @type {string} */ #token; - + /** @type {object} */ - #document; + #documentConfig; /** @type {string} */ #sessionId; @@ -67,7 +67,7 @@ class Telemetry { static BATCH_SIZE = 50; /** @type {number} */ - static FLUSH_INTERVAL = 5000; // 30 seconds + static FLUSH_INTERVAL = 10000; // 30 seconds /** @type {string} */ static COMMUNITY_DSN = 'https://public@telemetry.superdoc.dev/community'; @@ -100,17 +100,21 @@ class Telemetry { * @param {TelemetryConfig} config * @param {string} dsn - Data Source Name for telemetry service */ - async initTelemetry(config, dsn) { + initTelemetry(config, dsn) { const { projectId, token, endpoint } = this.#parseDsn(dsn); this.#projectId = projectId; this.#token = token; this.#endpoint = config.endpoint ?? endpoint; - this.#document = await this.processDocument(config.fileSource, { - id: config.documentId, - }); + this.#documentConfig = { + documentId: config.documentId, + internalId: config.internalId, + fileSource: config.fileSource, + } } - - + + /** + * Create source payload for request + */ getSourceData() { return { userAgent: window.navigator.userAgent, @@ -129,8 +133,13 @@ class Telemetry { * @param {string} name - Name of the feature/event * @param {Object.} [properties] - Additional properties */ - trackUsage(name, properties = {}) { + async trackUsage(name, properties = {}) { if (!this.#enabled) return; + + const document = await this.processDocument(this.#documentConfig.fileSource, { + id: this.#documentConfig.documentId, + internalId: properties.internalId, + }); /** @type {UsageEvent & BaseEvent} */ const event = { @@ -139,8 +148,8 @@ class Telemetry { timestamp: new Date().toISOString(), sessionId: this.#sessionId, source: this.getSourceData(), + document, name, - document: this.#document, properties, }; @@ -154,9 +163,15 @@ class Telemetry { * @param {string} path - Document path where item was found * @param {Object.} [metadata] - Additional context */ - trackParsing(category, name, path, metadata) { + async trackParsing(category, name, path, metadata) { if (!this.#enabled) return; + const document = await this.processDocument(this.#documentConfig.fileSource, { + id: this.#documentConfig.documentId, + internalId: metadata.internalId, + }); + + /** @type {ParsingEvent & BaseEvent} */ const event = { id: this.#generateId(), @@ -165,9 +180,9 @@ class Telemetry { sessionId: this.#sessionId, source: this.getSourceData(), category, + document, name, path, - document: this.#document, ...(metadata && { metadata }), }; @@ -181,24 +196,20 @@ class Telemetry { * @returns {Promise} Document metadata */ async processDocument(file, options = {}) { + let hash = ''; try { - const hash = await this.#generateMD5Hash(file); - - return { - id: options.id, - type: options.type || file.type, - internalId: options.internalId, - hash, - lastModified: file.lastModified ? new Date(file.lastModified).toISOString() : null, - }; + hash = await this.#generateMD5Hash(file); } catch (error) { - return { - id: options.id, - type: options.type || file.type, - internalId: options.internalId, - lastModified: file.lastModified ? new Date(file.lastModified).toISOString() : null, - } + console.error('Failed to retrieve file hash:', error); } + + return { + id: options.id, + type: options.type || file.type, + internalId: options.internalId, + hash, + lastModified: file.lastModified ? new Date(file.lastModified).toISOString() : null, + }; } /** @@ -232,11 +243,11 @@ class Telemetry { * @returns {Promise} */ async flush() { - if (!this.#enabled || this.#events.length === 0) return; + if (!this.#enabled || !this.#events.length) return; const eventsToSend = [...this.#events]; this.#events = []; - + try { const response = await fetch(this.#endpoint, { method: 'POST', diff --git a/packages/super-editor/src/core/helpers/ErrorWithDetails.js b/packages/super-editor/src/core/helpers/ErrorWithDetails.js new file mode 100644 index 0000000000..7c5d85e299 --- /dev/null +++ b/packages/super-editor/src/core/helpers/ErrorWithDetails.js @@ -0,0 +1,21 @@ +/** + * Custom error class which allows to pass additional details + * @param {string} name - Error name + * @param {string} message - Error message + * @param {object} details - additional details from the calling context + */ + +function ErrorWithDetails(name, message, details) { + this.name = name; + this.message = message || ''; + this.details = details; + + const error = new Error(this.message); + this.stack = error.stack; + error.name = this.name; +} +ErrorWithDetails.prototype = Object.create(Error.prototype); + +export { + ErrorWithDetails, +} diff --git a/packages/super-editor/src/core/super-converter/SuperConverter.js b/packages/super-editor/src/core/super-converter/SuperConverter.js index 8376db4141..20610af7ef 100644 --- a/packages/super-editor/src/core/super-converter/SuperConverter.js +++ b/packages/super-editor/src/core/super-converter/SuperConverter.js @@ -97,9 +97,7 @@ class SuperConverter { // Initialize telemetry this.telemetry = params?.telemetry || null; - - // ToDo do we still need to track unknown marks for telemetry - this.unknownMarks = []; + this.documentInternalId = null; // Parse the initial XML, if provided if (this.docx.length || this.xml) this.parseFromXml(); @@ -161,6 +159,13 @@ class SuperConverter { return { fontSizePt, kern, typeface, panose }; } } + + getDocumentInternalId() { + const settings = this.convertedXml['word/settings.xml']; + if (!settings) return ''; + const w15DocId = settings.elements[0].elements.find((el) => el.name === 'w15:docId'); + this.documentInternalId = w15DocId?.attributes['w15:val']; + } getThemeInfo(themeName) { themeName = themeName.toLowerCase(); @@ -181,6 +186,7 @@ class SuperConverter { } getSchema() { + this.getDocumentInternalId(); const result = createDocumentJson({...this.convertedXml, media: this.media }, this ); if (result) { diff --git a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js index e930a48f5a..bdeb7cc719 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js @@ -57,6 +57,7 @@ export const createDocumentJson = (docx, converter) => { if (result.content.length > 1) { converter.telemetry?.trackUsage('document_import', { documentType: 'docx', + internalId: converter?.documentInternalId, timestamp: new Date().toISOString() }); } @@ -137,7 +138,8 @@ const createNodeListHandler = (nodeHandlers) => { if (!nodesToHandle || nodesToHandle.length === 0) { converter?.telemetry?.trackParsing('node', 'empty_slice', `/word/${filename || 'document.xml'}`, { index, - totalElements: elements.length + internalId: converter?.documentInternalId, + totalElements: elements.length, }); continue; } @@ -169,7 +171,8 @@ const createNodeListHandler = (nodeHandlers) => { }); converter?.telemetry?.trackParsing('node', 'unhandled', `/word/${filename || 'document.xml'}`, { - context + context, + internalId: converter?.documentInternalId, }); continue; @@ -182,7 +185,8 @@ const createNodeListHandler = (nodeHandlers) => { converter?.telemetry?.trackParsing('node', 'invalid_index', `/word/${filename || 'document.xml'}`, { originalIndex: index - (consumed - 1), consumed, - resultingIndex: index + resultingIndex: index, + internalId: converter?.documentInternalId, }); index = 0; // Reset to safe value } @@ -196,7 +200,8 @@ const createNodeListHandler = (nodeHandlers) => { if (node.type === 'text' && Array.isArray(node.content) && !node.content.length) { converter?.telemetry?.trackParsing('node', 'empty_text', `/word/${filename || 'document.xml'}`, { - context: node.attrs + context: node.attrs, + internalId: converter?.documentInternalId, }); continue; } @@ -207,14 +212,20 @@ const createNodeListHandler = (nodeHandlers) => { } } } catch (error) { + const context = getSafeElementContext(elements, index); + if (error.details) { + context.elementAttributes = error.details; + } + // Track individual element processing errors with safe context converter?.telemetry?.trackParsing('element', 'processing_error', `/word/${filename || 'document.xml'}`, { error: { message: error.message, name: error.name, - stack: error.stack + stack: error.stack, }, - context: getSafeElementContext(elements, index) + internalId: converter?.documentInternalId, + context }); } } @@ -230,10 +241,11 @@ const createNodeListHandler = (nodeHandlers) => { name: error.name, stack: error.stack }, + internalId: converter?.documentInternalId, context: { totalElements: elements?.length || 0, processedCount: processedElements.length, - unhandledCount: unhandledNodes.length + unhandledCount: unhandledNodes.length, } }); diff --git a/packages/super-editor/src/core/super-converter/v2/importer/listImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/listImporter.js index f53e7bf0ef..8709225ee6 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/listImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/listImporter.js @@ -2,6 +2,7 @@ import { carbonCopy } from '../../../utilities/carbonCopy.js'; import { hasTextNode, parseProperties } from './importerHelpers.js'; import { preProcessNodesForFldChar } from './paragraphNodeImporter.js'; import { mergeTextNodes } from './mergeTextNodes.js'; +import { ErrorWithDetails } from '../../../helpers/ErrorWithDetails.js'; /** * @type {import("docxImporter").NodeHandler} @@ -428,7 +429,10 @@ export function getNodeNumberingDefinition(attributes, level, docx) { if (unorderedListTypes.includes(listTypeDef?.toLowerCase())) listType = 'bulletList'; else if (orderedListTypes.includes(listTypeDef)) listType = 'orderedList'; else { - throw new Error(`Unknown list type found during import: ${listTypeDef}`); + throw new ErrorWithDetails('ListParsingError', `Unknown list type found during import: ${listTypeDef}`, { + listOrderingType: listTypeDef, ilvl, numId, listrPrs, listpPrs, start, lvlText, lvlJc + }); + // throw new Error(`Unknown list type found during import: ${listTypeDef}`); } return { listType, listOrderingType: listTypeDef, ilvl, numId, listrPrs, listpPrs, start, lvlText, lvlJc }; From 71f49d8a6670f8af1585e5d42c49d90571b57a66 Mon Sep 17 00:00:00 2001 From: Vladyslava Andrushchenko Date: Thu, 23 Jan 2025 15:26:59 +0200 Subject: [PATCH 5/9] update dependencies --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 52a07d187a..c1fdf4a553 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10054,7 +10054,7 @@ }, "packages/superdoc": { "name": "@harbour-enterprises/superdoc", - "version": "0.3.8", + "version": "0.4.4", "license": "AGPL-3.0", "dependencies": { "@harbour-enterprises/super-editor": "0.0.1-alpha.0", From 6c4c34a788c53405bd3600204576de6ca36c4615 Mon Sep 17 00:00:00 2001 From: Vladyslava Andrushchenko Date: Thu, 23 Jan 2025 15:29:08 +0200 Subject: [PATCH 6/9] cleanup --- packages/super-editor/src/core/Editor.js | 7 ------- packages/super-editor/src/core/Telemetry.js | 2 +- .../src/core/super-converter/SuperConverter.js | 5 ----- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/packages/super-editor/src/core/Editor.js b/packages/super-editor/src/core/Editor.js index 18cae61141..7cb4ef46dc 100644 --- a/packages/super-editor/src/core/Editor.js +++ b/packages/super-editor/src/core/Editor.js @@ -580,13 +580,6 @@ export class Editor extends EventEmitter { }); } - /** - * Get document internal ID(w15:docId) from settings.xml - */ - initDocInternalId() { - this.options.documentInternalId = this.converter.getDocumentInternalId(); - } - /** * Initialize default styles for the editor container and ProseMirror. * Get page size and margins from the converter. diff --git a/packages/super-editor/src/core/Telemetry.js b/packages/super-editor/src/core/Telemetry.js index 508df91f57..6d50ed5540 100644 --- a/packages/super-editor/src/core/Telemetry.js +++ b/packages/super-editor/src/core/Telemetry.js @@ -67,7 +67,7 @@ class Telemetry { static BATCH_SIZE = 50; /** @type {number} */ - static FLUSH_INTERVAL = 10000; // 30 seconds + static FLUSH_INTERVAL = 30000; // 30 seconds /** @type {string} */ static COMMUNITY_DSN = 'https://public@telemetry.superdoc.dev/community'; diff --git a/packages/super-editor/src/core/super-converter/SuperConverter.js b/packages/super-editor/src/core/super-converter/SuperConverter.js index 20610af7ef..7f16ecadcd 100644 --- a/packages/super-editor/src/core/super-converter/SuperConverter.js +++ b/packages/super-editor/src/core/super-converter/SuperConverter.js @@ -256,11 +256,6 @@ class SuperConverter { this.media = this.convertedXml.media; this.addedMedia = processedData; } - - destroy() { - // Clean up telemetry when converter is destroyed - return this.telemetry?.destroy(); - } } export { SuperConverter }; From 3d55b5160d99811f10da057aca9adcbe02000a65 Mon Sep 17 00:00:00 2001 From: Vladyslava Andrushchenko Date: Fri, 24 Jan 2025 19:15:44 +0200 Subject: [PATCH 7/9] move telemetry init into SuperDoc --- package-lock.json | 9 + packages/super-editor/src/core/Editor.js | 31 +-- .../src/core/helpers/createDocument.js | 5 +- .../core/super-converter/SuperConverter.js | 10 +- .../v2/importer/docxImporter.js | 134 +++++++++---- .../v2/importer/imageImporter.js | 2 +- .../v2/importer/paragraphNodeImporter.js | 4 +- .../v2/importer/runNodeImporter.js | 2 +- .../v2/importer/standardNodeImporter.js | 4 +- .../dev/components/DeveloperPlayground.vue | 12 +- packages/superdoc/package.json | 1 + packages/superdoc/src/SuperDoc.vue | 5 + packages/superdoc/src/core/SuperDoc.js | 18 ++ .../src/core => shared/common}/Telemetry.js | 186 ++++++++---------- shared/common/index.js | 1 + 15 files changed, 235 insertions(+), 189 deletions(-) rename {packages/super-editor/src/core => shared/common}/Telemetry.js (56%) diff --git a/package-lock.json b/package-lock.json index c1fdf4a553..716aa85bfb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2569,6 +2569,14 @@ "ieee754": "^1.1.13" } }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -10058,6 +10066,7 @@ "license": "AGPL-3.0", "dependencies": { "@harbour-enterprises/super-editor": "0.0.1-alpha.0", + "buffer-crc32": "^1.0.0", "eventemitter3": "^5.0.1", "jsdom": "^25.0.1", "naive-ui": "^2.39.0", diff --git a/packages/super-editor/src/core/Editor.js b/packages/super-editor/src/core/Editor.js index 7cb4ef46dc..83ff075c54 100644 --- a/packages/super-editor/src/core/Editor.js +++ b/packages/super-editor/src/core/Editor.js @@ -15,7 +15,6 @@ import { trackedTransaction } from '@extensions/track-changes/trackChangesHelper import { TrackChangesBasePluginKey } from '@extensions/track-changes/plugins/index.js'; import { initPaginationData, PaginationPluginKey } from '@extensions/pagination/pagination-helpers'; import DocxZipper from '@core/DocxZipper.js'; -import { Telemetry } from '@core/Telemetry.js'; /** * Editor main class. @@ -80,13 +79,12 @@ export class Editor extends EventEmitter { onDocumentLocked: () => null, onFirstRender: () => null, onCollaborationReady: () => null, + onExceptionCaught: () => null, // async (file) => url; handleImageUpload: null, - - + // telemetry telemetry: null, - telemetryService: null, }; constructor(options) { @@ -105,12 +103,11 @@ export class Editor extends EventEmitter { }; let initMode = modes[this.options.mode] ?? modes.default; - + initMode(); } #init(options) { - this.#initTelemetry(); this.#createExtensionService(); this.#createCommandService(); this.#createSchema(); @@ -120,6 +117,7 @@ export class Editor extends EventEmitter { this.on('beforeCreate', this.options.onBeforeCreate); this.emit('beforeCreate', { editor: this }); this.on('contentError', this.options.onContentError); + this.on('exceptionCaught', this.options.onExceptionCaught); this.#createView(); this.initDefaultStyles(); @@ -445,20 +443,7 @@ export class Editor extends EventEmitter { docx: this.options.content, media: this.options.mediaFiles, debug: true, - telemetry: this.telemetryService, - }); - } - } - - /** - * Initialize telemetry service. - */ - async #initTelemetry() { - if (this.options.telemetry?.enabled) { - this.telemetryService = new Telemetry({ - enabled: this.options.telemetry.enabled ?? true, - dsn: this.options.telemetry?.dsn, - endpoint: this.options.telemetry.endpoint, + telemetry: this.options.telemetry, fileSource: this.options.fileSource, documentId: this.options.documentId, }); @@ -520,7 +505,7 @@ export class Editor extends EventEmitter { const { mode, fragment, isHeadless, content, loadFromSchema } = this.options; if (mode === 'docx') { - doc = createDocument(this.converter, this.schema); + doc = createDocument(this.converter, this.schema, this); if (fragment && isHeadless) { doc = yXmlFragmentToProseMirrorRootNode(fragment, this.schema); @@ -862,7 +847,7 @@ export class Editor extends EventEmitter { isHeadless: this.options.isHeadless, }); - this.telemetryService?.trackUsage('document_export', { + this.telemetry?.trackUsage('document_export', { documentType: 'docx', timestamp: new Date().toISOString() }); @@ -890,6 +875,6 @@ export class Editor extends EventEmitter { this.removeAllListeners(); // Clean up telemetry when editor is destroyed - this.telemetry?.destroy(); + // this.telemetry?.destroy(); } } diff --git a/packages/super-editor/src/core/helpers/createDocument.js b/packages/super-editor/src/core/helpers/createDocument.js index d6cf5007cb..177022cc86 100644 --- a/packages/super-editor/src/core/helpers/createDocument.js +++ b/packages/super-editor/src/core/helpers/createDocument.js @@ -2,10 +2,11 @@ * Creates the document to pass to EditorState. * @param converter SuperConverter instance. * @param schema Schema. + * @param editor Editor * @returns Document. */ -export function createDocument(converter, schema) { - const documentData = converter.getSchema(); +export function createDocument(converter, schema, editor) { + const documentData = converter.getSchema(editor); if (documentData) { return schema.nodeFromJSON(documentData); diff --git a/packages/super-editor/src/core/super-converter/SuperConverter.js b/packages/super-editor/src/core/super-converter/SuperConverter.js index 7f16ecadcd..e9a17ff31c 100644 --- a/packages/super-editor/src/core/super-converter/SuperConverter.js +++ b/packages/super-editor/src/core/super-converter/SuperConverter.js @@ -98,6 +98,10 @@ class SuperConverter { // Initialize telemetry this.telemetry = params?.telemetry || null; this.documentInternalId = null; + + // Uploaded file + this.fileSource = params?.fileSource || null; + this.documentId = params?.documentId || null; // Parse the initial XML, if provided if (this.docx.length || this.xml) this.parseFromXml(); @@ -163,6 +167,8 @@ class SuperConverter { getDocumentInternalId() { const settings = this.convertedXml['word/settings.xml']; if (!settings) return ''; + // New versions of Word will have w15:docId + // It's possible to have w14:docId as well but Word(2013 and later) will convert it automatically when document opened const w15DocId = settings.elements[0].elements.find((el) => el.name === 'w15:docId'); this.documentInternalId = w15DocId?.attributes['w15:val']; } @@ -185,9 +191,9 @@ class SuperConverter { return { typeface, panose }; } - getSchema() { + getSchema(editor) { this.getDocumentInternalId(); - const result = createDocumentJson({...this.convertedXml, media: this.media }, this ); + const result = createDocumentJson({...this.convertedXml, media: this.media }, this, editor); if (result) { this.savedTagsToRestore.push({ ...result.savedTagsToRestore }); diff --git a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js index bdeb7cc719..f36a0d503b 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js @@ -30,9 +30,11 @@ import { listHandlerEntity } from './listImporter.js'; /** * * @param {ParsedDocx} docx + * @param {SuperConverter} converter instance. + * @param {Editor} editor instance. * @returns {{pmDoc: PmNodeJson, savedTagsToRestore: XmlNode, pageStyles: *}|null} */ -export const createDocumentJson = (docx, converter) => { +export const createDocumentJson = (docx, converter, editor) => { const json = carbonCopy(getInitialJSON(docx)); if (!json) return null; @@ -44,7 +46,7 @@ export const createDocumentJson = (docx, converter) => { const ignoreNodes = ['w:sectPr']; const content = node.elements?.filter((n) => !ignoreNodes.includes(n.name)) ?? []; - const parsedContent = nodeListHandler.handler(content, docx, false, converter); + const parsedContent = nodeListHandler.handler(content, docx, false, converter, editor); const result = { type: 'doc', content: parsedContent, @@ -55,7 +57,11 @@ export const createDocumentJson = (docx, converter) => { // Not empty document if (result.content.length > 1) { - converter.telemetry?.trackUsage('document_import', { + converter?.telemetry?.trackUsage( + converter?.fileSource, + converter?.documentId, + 'document_import', + { documentType: 'docx', internalId: converter?.documentInternalId, timestamp: new Date().toISOString() @@ -65,7 +71,7 @@ export const createDocumentJson = (docx, converter) => { return { pmDoc: result, savedTagsToRestore: node, - pageStyles: getDocumentStyles(node, docx, converter), + pageStyles: getDocumentStyles(node, docx, converter, editor), }; } return null; @@ -125,7 +131,7 @@ const createNodeListHandler = (nodeHandlers) => { }; }; - const nodeListHandlerFn = (elements, docx, insideTrackChange, converter, filename) => { + const nodeListHandlerFn = (elements, docx, insideTrackChange, converter, editor, filename) => { if (!elements || !elements.length) return []; const processedElements = []; @@ -136,7 +142,13 @@ const createNodeListHandler = (nodeHandlers) => { try { const nodesToHandle = elements.slice(index); if (!nodesToHandle || nodesToHandle.length === 0) { - converter?.telemetry?.trackParsing('node', 'empty_slice', `/word/${filename || 'document.xml'}`, { + converter?.telemetry?.trackParsing( + converter?.fileSource, + converter?.documentId, + 'node', + 'empty_slice', + `/word/${filename || 'document.xml'}`, + { index, internalId: converter?.documentInternalId, totalElements: elements.length, @@ -154,6 +166,7 @@ const createNodeListHandler = (nodeHandlers) => { { handler: nodeListHandlerFn, handlerEntities: nodeHandlers }, insideTrackChange, converter, + editor, filename ); }, @@ -170,7 +183,13 @@ const createNodeListHandler = (nodeHandlers) => { attributes: context.elementAttributes }); - converter?.telemetry?.trackParsing('node', 'unhandled', `/word/${filename || 'document.xml'}`, { + converter?.telemetry?.trackParsing( + converter?.fileSource, + converter?.documentId, + 'node', + 'unhandled', + `/word/${filename || 'document.xml'}`, + { context, internalId: converter?.documentInternalId, }); @@ -182,12 +201,19 @@ const createNodeListHandler = (nodeHandlers) => { if (consumed > 0) { index += consumed - 1; if (index < 0) { - converter?.telemetry?.trackParsing('node', 'invalid_index', `/word/${filename || 'document.xml'}`, { - originalIndex: index - (consumed - 1), - consumed, - resultingIndex: index, - internalId: converter?.documentInternalId, - }); + converter?.telemetry?.trackParsing( + converter?.fileSource, + converter?.documentId, + 'node', + 'invalid_index', + `/word/${filename || 'document.xml'}`, + { + originalIndex: index - (consumed - 1), + consumed, + resultingIndex: index, + internalId: converter?.documentInternalId, + } + ); index = 0; // Reset to safe value } } @@ -199,7 +225,13 @@ const createNodeListHandler = (nodeHandlers) => { if (node.type === 'text' && Array.isArray(node.content) && !node.content.length) { - converter?.telemetry?.trackParsing('node', 'empty_text', `/word/${filename || 'document.xml'}`, { + converter?.telemetry?.trackParsing( + converter?.fileSource, + converter?.documentId, + 'node', + 'empty_text', + `/word/${filename || 'document.xml'}`, + { context: node.attrs, internalId: converter?.documentInternalId, }); @@ -212,43 +244,59 @@ const createNodeListHandler = (nodeHandlers) => { } } } catch (error) { + editor?.emit('exceptionCaught', { error }); + const context = getSafeElementContext(elements, index); if (error.details) { context.elementAttributes = error.details; } // Track individual element processing errors with safe context - converter?.telemetry?.trackParsing('element', 'processing_error', `/word/${filename || 'document.xml'}`, { - error: { - message: error.message, - name: error.name, - stack: error.stack, - }, - internalId: converter?.documentInternalId, - context - }); + converter?.telemetry?.trackParsing( + converter?.fileSource, + converter?.documentId, + 'element', + 'processing_error', + `/word/${filename || 'document.xml'}`, + { + error: { + message: error.message, + name: error.name, + stack: error.stack, + }, + internalId: converter?.documentInternalId, + context + } + ); } } return processedElements; - } catch (error) { + editor?.emit('exceptionCaught', { error }); + // Track catastrophic errors in the node list handler - converter?.telemetry?.trackParsing('handler', 'nodeListHandler', `/word/${filename || 'document.xml'}`, { - status: 'error', - error: { - message: error.message, - name: error.name, - stack: error.stack - }, - internalId: converter?.documentInternalId, - context: { - totalElements: elements?.length || 0, - processedCount: processedElements.length, - unhandledCount: unhandledNodes.length, + converter?.telemetry?.trackParsing( + converter?.fileSource, + converter?.documentId, + 'handler', + 'nodeListHandler', + `/word/${filename || 'document.xml'}`, + { + status: 'error', + error: { + message: error.message, + name: error.name, + stack: error.stack + }, + internalId: converter?.documentInternalId, + context: { + totalElements: elements?.length || 0, + processedCount: processedElements.length, + unhandledCount: unhandledNodes.length, + } } - }); - + ); throw error; } }; @@ -261,7 +309,7 @@ const createNodeListHandler = (nodeHandlers) => { * @param {XmlNode} node * @returns {Object} The document styles object */ -function getDocumentStyles(node, docx, converter) { +function getDocumentStyles(node, docx, converter, editor) { const sectPr = node.elements?.find((n) => n.name === 'w:sectPr'); const styles = {}; @@ -299,17 +347,17 @@ function getDocumentStyles(node, docx, converter) { }; break; case 'w:headerReference': - getHeaderFooter(el, 'header', docx, converter); + getHeaderFooter(el, 'header', docx, converter, editor); break; case 'w:footerReference': - getHeaderFooter(el, 'footer', docx, converter); + getHeaderFooter(el, 'footer', docx, converter, editor); break; } }); return styles; } -function getHeaderFooter(el, elementType, docx, converter) { +function getHeaderFooter(el, elementType, docx, converter, editor) { const rels = docx['word/_rels/document.xml.rels']; const relationships = rels.elements.find((el) => el.name === 'Relationships'); const { elements } = relationships; @@ -326,7 +374,7 @@ function getHeaderFooter(el, elementType, docx, converter) { const currentFileName = target; const nodeListHandler = defaultNodeListHandler(); - const schema = nodeListHandler.handler(referenceFile.elements[0].elements, docx, false, currentFileName); + const schema = nodeListHandler.handler(referenceFile.elements[0].elements, docx, false, converter, editor, currentFileName); let storage, storageIds; diff --git a/packages/super-editor/src/core/super-converter/v2/importer/imageImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/imageImporter.js index 291483f673..77499fb68e 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/imageImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/imageImporter.js @@ -3,7 +3,7 @@ import { emuToPixels } from '../../helpers.js'; /** * @type {import("docxImporter").NodeHandler} */ -export const handleDrawingNode = (nodes, docx, nodeListHandler, insideTrackChange, filename) => { +export const handleDrawingNode = (nodes, docx, nodeListHandler, insideTrackChange, converter, editor, filename) => { if (nodes.length === 0 || nodes[0].name !== 'w:drawing') { return { nodes: [], consumed: 0 }; } diff --git a/packages/super-editor/src/core/super-converter/v2/importer/paragraphNodeImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/paragraphNodeImporter.js index 9897d4f00f..72d5c890ee 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/paragraphNodeImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/paragraphNodeImporter.js @@ -11,7 +11,7 @@ import { mergeTextNodes } from './mergeTextNodes.js'; * * @type {import("docxImporter").NodeHandler} */ -export const handleParagraphNode = (nodes, docx, nodeListHandler, insideTrackChange, filename) => { +export const handleParagraphNode = (nodes, docx, nodeListHandler, insideTrackChange, converter, editor, filename) => { if (nodes.length === 0 || nodes[0].name !== 'w:p') { return { nodes: [], consumed: 0 }; } @@ -38,7 +38,7 @@ export const handleParagraphNode = (nodes, docx, nodeListHandler, insideTrackCha return { nodes: [], consumed: 0 }; } - const result = handleStandardNode([node], docx, nodeListHandler, insideTrackChange, filename); + const result = handleStandardNode([node], docx, nodeListHandler, insideTrackChange, converter, editor, filename); if (result.nodes.length === 1) { schemaNode = result.nodes[0]; } diff --git a/packages/super-editor/src/core/super-converter/v2/importer/runNodeImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/runNodeImporter.js index 9804e80e66..0866f78d15 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/runNodeImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/runNodeImporter.js @@ -4,7 +4,7 @@ import { createImportMarks } from './markImporter.js'; /** * @type {import("docxImporter").NodeHandler} */ -const handleRunNode = (nodes, docx, nodeListHandler, insideTrackChange = false, filename) => { +const handleRunNode = (nodes, docx, nodeListHandler, insideTrackChange = false, converter, editor, filename) => { if (nodes.length === 0 || nodes[0].name !== 'w:r') { return { nodes: [], consumed: 0 }; } diff --git a/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js index eb95d00329..720eb56773 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js @@ -3,7 +3,7 @@ import { getElementName, parseProperties } from './importerHelpers.js'; /** * @type {import("docxImporter").NodeHandler} */ -export const handleStandardNode = (nodes, docx, nodeListHandler, insideTrackChange = false, filename) => { +export const handleStandardNode = (nodes, docx, nodeListHandler, insideTrackChange = false, converter, editor, filename) => { if (!nodes || nodes.length === 0) { return { nodes: [], consumed: 0 }; } @@ -34,7 +34,7 @@ export const handleStandardNode = (nodes, docx, nodeListHandler, insideTrackChan el.marks.push(...marks); return el; }); - content.push(...nodeListHandler.handler(updatedElements, docx, insideTrackChange, filename)); + content.push(...nodeListHandler.handler(updatedElements, docx, insideTrackChange, converter, editor, filename)); } const resultNode = { diff --git a/packages/super-editor/src/dev/components/DeveloperPlayground.vue b/packages/super-editor/src/dev/components/DeveloperPlayground.vue index 38ea28a510..a299b52589 100644 --- a/packages/super-editor/src/dev/components/DeveloperPlayground.vue +++ b/packages/super-editor/src/dev/components/DeveloperPlayground.vue @@ -8,16 +8,17 @@ import { SuperEditor } from '@/index.js'; import { getFileObject } from '@harbour-enterprises/common/helpers/get-file-object'; import { DOCX } from '@harbour-enterprises/common'; import { SuperToolbar } from '@components/toolbar/super-toolbar'; -import { fieldAnnotationHelpers } from '@extensions/index.js'; import { PaginationPluginKey } from '@extensions/pagination/pagination-helpers.js'; import BasicUpload from './BasicUpload.vue'; import BlankDOCX from '@harbour-enterprises/common/data/blank.docx?url'; +import { Telemetry } from '@harbour-enterprises/common/Telemetry.js'; // Import the component the same you would in your app let activeEditor; const currentFile = ref(null); const pageStyles = ref(null); const isDebuggingPagination = ref(false); +const telemetry = ref(null); const handleNewFile = async (file) => { currentFile.value = null; @@ -65,9 +66,7 @@ const editorOptions = computed(() => { suppressSkeletonLoader: true, users: [], // For comment @-mentions, only users that have access to the document pagination: true, - telemetry: { - enabled: true - } + telemetry: telemetry.value, } }); @@ -109,6 +108,11 @@ const debugPageStyle = computed(() => { onMounted(async () => { // set document to blank currentFile.value = await getFileObject(BlankDOCX, 'blank_document.docx', DOCX); + + telemetry.value = new Telemetry({ + enabled: true, + superdocId: 'dev-playground', + }); }); diff --git a/packages/superdoc/package.json b/packages/superdoc/package.json index 3a60b5b2e2..5d2545e30f 100644 --- a/packages/superdoc/package.json +++ b/packages/superdoc/package.json @@ -39,6 +39,7 @@ }, "dependencies": { "@harbour-enterprises/super-editor": "0.0.1-alpha.0", + "buffer-crc32": "^1.0.0", "eventemitter3": "^5.0.1", "jsdom": "^25.0.1", "naive-ui": "^2.39.0", diff --git a/packages/superdoc/src/SuperDoc.vue b/packages/superdoc/src/SuperDoc.vue index 2b9d2c8880..18846e1489 100644 --- a/packages/superdoc/src/SuperDoc.vue +++ b/packages/superdoc/src/SuperDoc.vue @@ -272,6 +272,10 @@ const onEditorContentError = ({ error, editor }) => { proxy.$superdoc.emit('content-error', { error, editor }); }; +const onEditorExceptionCaught = ({ error, editor }) => { + proxy.$superdoc.emit('exception-caught', { error, editor }); +}; + const updateToolbarState = () => { proxy.$superdoc.toolbar.updateToolbarState(); }; @@ -296,6 +300,7 @@ const editorOptions = (doc) => { onSelectionUpdate: onEditorSelectionChange, onCollaborationReady: onEditorCollaborationReady, onContentError: onEditorContentError, + onExceptionCaught: onEditorExceptionCaught, // onCommentsLoaded, // onCommentClicked, // onCommentsUpdate: onEditorCommentsUpdate, diff --git a/packages/superdoc/src/core/SuperDoc.js b/packages/superdoc/src/core/SuperDoc.js index 2f9ab60bfc..cd94e698c5 100644 --- a/packages/superdoc/src/core/SuperDoc.js +++ b/packages/superdoc/src/core/SuperDoc.js @@ -12,6 +12,7 @@ import { SuperToolbar } from '@harbour-enterprises/super-editor'; import { createAwarenessHandler, createProvider } from './collaboration/collaboration'; import { createSuperdocVueApp } from './create-app'; import { shuffleArray } from '@harbour-enterprises/common/collaboration/awareness.js'; +import { Telemetry } from '@harbour-enterprises/common/Telemetry.js'; /** * @typedef {Object} SuperdocUser The current user of this superdoc @@ -71,6 +72,7 @@ export class SuperDoc extends EventEmitter { onPdfDocumentReady: () => null, onSidebarToggle: () => null, onCollaborationReady: () => null, + onExceptionCaught: () => null, // Image upload handler // async (file) => url; @@ -98,6 +100,8 @@ export class SuperDoc extends EventEmitter { // Initialize collaboration if configured await this.#initCollaboration(this.config.modules); + + this.#initTelemetry(); this.#initVueApp(); this.#initListeners(); @@ -161,6 +165,7 @@ export class SuperDoc extends EventEmitter { this.on('sidebar-toggle', this.config.onSidebarToggle); this.on('collaboration-ready', this.config.onCollaborationReady); this.on('content-error', this.onContentError); + this.on('exception-caught', this.config.onExceptionCaught); } /* ** @@ -210,6 +215,19 @@ export class SuperDoc extends EventEmitter { return processedDocuments; } + /** + * Initialize telemetry service. + */ + #initTelemetry() { + if (this.config.telemetry?.enabled) { + this.telemetry = new Telemetry({ + enabled: this.config.telemetry.enabled ?? true, + licenceKey: this.config.telemetry.licenceKey, + superdocId: this.superdocId, + }); + } + } + onContentError({ error, editor }) { const { documentId } = editor.options; const doc = this.superdocStore.documents.find((d) => d.id === documentId); diff --git a/packages/super-editor/src/core/Telemetry.js b/shared/common/Telemetry.js similarity index 56% rename from packages/super-editor/src/core/Telemetry.js rename to shared/common/Telemetry.js index 6d50ed5540..5969358d1a 100644 --- a/packages/super-editor/src/core/Telemetry.js +++ b/shared/common/Telemetry.js @@ -3,24 +3,27 @@ * @property {string} id - Unique event identifier * @property {string} timestamp - ISO timestamp of the event * @property {string} sessionId - Current session identifier + * @property {string} superdocId - SuperDoc ID * @property {Object} document - Document information * @property {string} [document.id] - Reference ID * @property {string} [document.type] - Document type * @property {string} [document.internalId] - Internal document ID - * @property {string} [document.hash] - Document MD5 hash + * @property {string} [document.hash] - Document CRC32 hash * @property {string} [document.lastModified] - Last modified timestamp */ /** * @typedef {Object} UsageEvent - * @property {'usage'} type - Event type identifier + * @param {File} fileSource - File object + * @param {string} documentId - document id * @property {string} name - Name of the usage event * @property {Object.} properties - Event properties */ /** * @typedef {Object} ParsingEvent - * @property {'parsing'} type - Event type identifier + * @param {File} fileSource - File object + * @param {string} documentId - document id * @property {'mark'|'element'} category - Category of the parsed item * @property {string} name - Name of the parsed item * @property {string} path - Document path where item was found @@ -31,84 +34,63 @@ /** * @typedef {Object} TelemetryConfig - * @property {string} [dsn] - Data Source Name for telemetry service - * @property {string} [endpoint] - Optional override for telemetry endpoint + * @property {string} [licenceKey] - Licence key for telemetry service * @property {boolean} [enabled=true] - Whether telemetry is enabled - * @property {File} [fileSource] - current editor file - * @property {string} documentId - Reference ID + * @property {string} endpoint - service endpoint + * @property {string} superdocId - SuperDoc id */ +import crc32 from 'buffer-crc32'; + class Telemetry { /** @type {boolean} */ - #enabled; + enabled; /** @type {string} */ - #endpoint; + superdocId; /** @type {string} */ - #projectId; + licenseKey; /** @type {string} */ - #token; - - /** @type {object} */ - #documentConfig; + endpoint; /** @type {string} */ - #sessionId; + sessionId; /** @type {TelemetryEvent[]} */ - #events = []; + events = []; /** @type {number|undefined} */ - #flushInterval; + flushInterval; /** @type {number} */ static BATCH_SIZE = 50; /** @type {number} */ - static FLUSH_INTERVAL = 30000; // 30 seconds + static FLUSH_INTERVAL = 10000; // 30 seconds + + /** @type {string} */ + static COMMUNITY_LICENSE_KEY = 'community-and-eval-agplv3'; /** @type {string} */ - static COMMUNITY_DSN = 'https://public@telemetry.superdoc.dev/community'; + static DEFAULT_ENDPOINT = 'https://ingest.superdoc.dev/v1/collect'; /** * Initialize telemetry service * @param {TelemetryConfig} config */ constructor(config = {}) { - this.#enabled = config.enabled ?? true; + this.enabled = config.enabled ?? true; - try { - const dsn = config.dsn ?? Telemetry.COMMUNITY_DSN; - this.initTelemetry(config, dsn); - } catch (error) { - console.warn('Invalid telemetry configuration, enabling community version:', error); - if (config.dsn) this.initTelemetry(config, Telemetry.COMMUNITY_DSN); - return; - } + this.licenseKey = config.licenceKey ?? Telemetry.COMMUNITY_LICENSE_KEY; + this.endpoint = config.endpoint ?? Telemetry.DEFAULT_ENDPOINT; + this.superdocId = config.superdocId; - this.#sessionId = this.#generateId(); + this.sessionId = this.generateId(); - if (this.#enabled) { - this.#startPeriodicFlush(); - } - } - - /** - * Init variables - * @param {TelemetryConfig} config - * @param {string} dsn - Data Source Name for telemetry service - */ - initTelemetry(config, dsn) { - const { projectId, token, endpoint } = this.#parseDsn(dsn); - this.#projectId = projectId; - this.#token = token; - this.#endpoint = config.endpoint ?? endpoint; - this.#documentConfig = { - documentId: config.documentId, - internalId: config.internalId, - fileSource: config.fileSource, + if (this.enabled) { + this.startPeriodicFlush(); } } @@ -130,63 +112,68 @@ class Telemetry { /** * Track feature usage + * @param {File} fileSource - File object + * @param {string} documentId - document id * @param {string} name - Name of the feature/event * @param {Object.} [properties] - Additional properties */ - async trackUsage(name, properties = {}) { - if (!this.#enabled) return; - - const document = await this.processDocument(this.#documentConfig.fileSource, { - id: this.#documentConfig.documentId, + async trackUsage(fileSource, documentId, name, properties = {}) { + if (!this.enabled) return; + + const processedDoc = await this.processDocument(fileSource, { + id: documentId, internalId: properties.internalId, }); /** @type {UsageEvent & BaseEvent} */ const event = { - id: this.#generateId(), + id: this.generateId(), type: 'usage', timestamp: new Date().toISOString(), - sessionId: this.#sessionId, + sessionId: this.sessionId, + superdocId: this.superdocId, source: this.getSourceData(), - document, + document: processedDoc, name, properties, }; - this.#queueEvent(event); + this.queueEvent(event); } /** * Track parsing events + * @param {File} fileSource - File object + * @param {string} documentId - document id * @param {'mark'|'element'} category - Category of parsed item * @param {string} name - Name of the item * @param {string} path - Document path where item was found * @param {Object.} [metadata] - Additional context */ - async trackParsing(category, name, path, metadata) { - if (!this.#enabled) return; + async trackParsing(fileSource, documentId, category, name, path, metadata) { + if (!this.enabled) return; - const document = await this.processDocument(this.#documentConfig.fileSource, { - id: this.#documentConfig.documentId, + const processedDoc = await this.processDocument(fileSource, { + id: documentId, internalId: metadata.internalId, }); - /** @type {ParsingEvent & BaseEvent} */ const event = { - id: this.#generateId(), + id: this.generateId(), type: 'parsing', timestamp: new Date().toISOString(), - sessionId: this.#sessionId, + sessionId: this.sessionId, + superdocId: this.superdocId, source: this.getSourceData(), category, - document, + document: processedDoc, name, path, ...(metadata && { metadata }), }; - this.#queueEvent(event); + this.queueEvent(event); } /** @@ -198,7 +185,7 @@ class Telemetry { async processDocument(file, options = {}) { let hash = ''; try { - hash = await this.#generateMD5Hash(file); + hash = await this.generateCrc32Hash(file); } catch (error) { console.error('Failed to retrieve file hash:', error); } @@ -213,15 +200,16 @@ class Telemetry { } /** - * Generate MD5 hash for a file + * Generate CRC32 hash for a file * @param {File} file - File to hash - * @returns {Promise} MD5 hash + * @returns {Promise} CRC32 hash * @private */ - async #generateMD5Hash(file) { - const buffer = await file.arrayBuffer(); - const hashBuffer = await crypto.subtle.digest('SHA-256', buffer); - const hashArray = Array.from(new Uint8Array(hashBuffer)); + async generateCrc32Hash(file) { + const arrayBuffer = await file.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + const hashBuffer = crc32(buffer); + const hashArray = Array.from(hashBuffer); return hashArray.map((b) => b.toString(16).padStart(2, '0')).join(''); } @@ -230,10 +218,10 @@ class Telemetry { * @param {TelemetryEvent} event * @private */ - #queueEvent(event) { - this.#events.push(event); + queueEvent(event) { + this.events.push(event); - if (this.#events.length >= Telemetry.BATCH_SIZE) { + if (this.events.length >= Telemetry.BATCH_SIZE) { this.flush(); } } @@ -243,18 +231,17 @@ class Telemetry { * @returns {Promise} */ async flush() { - if (!this.#enabled || !this.#events.length) return; + if (!this.enabled || !this.events.length) return; - const eventsToSend = [...this.#events]; - this.#events = []; + const eventsToSend = [...this.events]; + this.events = []; try { - const response = await fetch(this.#endpoint, { + const response = await fetch(this.endpoint, { method: 'POST', headers: { 'Content-Type': 'application/json', - 'X-Project-ID': this.#projectId, - 'X-Token': this.#token, + 'X-License-Key': this.licenseKey, }, body: JSON.stringify(eventsToSend), }); @@ -265,7 +252,7 @@ class Telemetry { } catch (error) { console.error('Failed to upload telemetry:', error); // Add events back to queue - this.#events = [...eventsToSend, ...this.#events]; + this.events = [...eventsToSend, ...this.events]; } } @@ -273,39 +260,20 @@ class Telemetry { * Start periodic flush interval * @private */ - #startPeriodicFlush() { - this.#flushInterval = setInterval(() => { - if (this.#events.length > 0) { + startPeriodicFlush() { + this.flushInterval = setInterval(() => { + if (this.events.length > 0) { this.flush(); } }, Telemetry.FLUSH_INTERVAL); } - - /** - * Parse DSN string into config - * @param {string} dsn - * @returns {{ projectId: string, token: string, endpoint: string }} - * @private - */ - #parseDsn(dsn) { - try { - const url = new URL(dsn); - return { - token: url.username, - projectId: url.pathname.split('/')[1], - endpoint: `${url.protocol}//${url.host}/v1/collect`, - }; - } catch (error) { - throw new Error(`Invalid DSN: ${error.message}`); - } - } - + /** * Generate unique identifier * @returns {string} * @private */ - #generateId() { + generateId() { return `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; } @@ -314,8 +282,8 @@ class Telemetry { * @returns {Promise} */ destroy() { - if (this.#flushInterval) { - clearInterval(this.#flushInterval); + if (this.flushInterval) { + clearInterval(this.flushInterval); } return this.flush(); } diff --git a/shared/common/index.js b/shared/common/index.js index b2bb4cdf3f..df1af6617c 100644 --- a/shared/common/index.js +++ b/shared/common/index.js @@ -5,3 +5,4 @@ export * from './helpers/get-file-object.js'; export * from './helpers/compare-superdoc-versions.js'; export { default as vClickOutside } from './helpers/v-click-outside.js'; export { default as BasicUpload } from './components/BasicUpload.vue'; +export { Telemetry } from './Telemetry.js'; From 8b55333cb29b752399e8c45aa553f1605c9f78ec Mon Sep 17 00:00:00 2001 From: Caio Pizzol <97641911+caio-pizzol@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:38:17 -0300 Subject: [PATCH 8/9] Potential fix for code scanning alert no. 8: Insecure randomness LGTM Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- shared/common/Telemetry.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shared/common/Telemetry.js b/shared/common/Telemetry.js index 5969358d1a..bb0d17402c 100644 --- a/shared/common/Telemetry.js +++ b/shared/common/Telemetry.js @@ -41,6 +41,7 @@ */ import crc32 from 'buffer-crc32'; +import { randomBytes } from 'crypto'; class Telemetry { /** @type {boolean} */ @@ -274,7 +275,8 @@ class Telemetry { * @private */ generateId() { - return `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; + const randomValue = randomBytes(4).toString('hex'); + return `${Date.now()}-${randomValue}`; } /** From 592d34f877d3d4afc6663f75293ff238d78ebb34 Mon Sep 17 00:00:00 2001 From: Vladyslava Andrushchenko Date: Fri, 24 Jan 2025 22:53:34 +0200 Subject: [PATCH 9/9] review comments --- packages/super-editor/src/core/Editor.js | 7 ++----- .../src/core/super-converter/v2/importer/docxImporter.js | 4 ++-- packages/superdoc/src/SuperDoc.vue | 6 +++--- packages/superdoc/src/core/SuperDoc.js | 7 +++++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/super-editor/src/core/Editor.js b/packages/super-editor/src/core/Editor.js index 83ff075c54..d39c86e6df 100644 --- a/packages/super-editor/src/core/Editor.js +++ b/packages/super-editor/src/core/Editor.js @@ -79,7 +79,7 @@ export class Editor extends EventEmitter { onDocumentLocked: () => null, onFirstRender: () => null, onCollaborationReady: () => null, - onExceptionCaught: () => null, + onException: () => null, // async (file) => url; handleImageUpload: null, @@ -117,7 +117,7 @@ export class Editor extends EventEmitter { this.on('beforeCreate', this.options.onBeforeCreate); this.emit('beforeCreate', { editor: this }); this.on('contentError', this.options.onContentError); - this.on('exceptionCaught', this.options.onExceptionCaught); + this.on('exception', this.options.onException); this.#createView(); this.initDefaultStyles(); @@ -873,8 +873,5 @@ export class Editor extends EventEmitter { if (this.view) this.view.destroy(); this.#endCollaboration(); this.removeAllListeners(); - - // Clean up telemetry when editor is destroyed - // this.telemetry?.destroy(); } } diff --git a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js index f36a0d503b..bd85fee99b 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js @@ -244,7 +244,7 @@ const createNodeListHandler = (nodeHandlers) => { } } } catch (error) { - editor?.emit('exceptionCaught', { error }); + editor?.emit('exception', { error }); const context = getSafeElementContext(elements, index); if (error.details) { @@ -273,7 +273,7 @@ const createNodeListHandler = (nodeHandlers) => { return processedElements; } catch (error) { - editor?.emit('exceptionCaught', { error }); + editor?.emit('exception', { error }); // Track catastrophic errors in the node list handler converter?.telemetry?.trackParsing( diff --git a/packages/superdoc/src/SuperDoc.vue b/packages/superdoc/src/SuperDoc.vue index 18846e1489..85de5fa9b7 100644 --- a/packages/superdoc/src/SuperDoc.vue +++ b/packages/superdoc/src/SuperDoc.vue @@ -272,8 +272,8 @@ const onEditorContentError = ({ error, editor }) => { proxy.$superdoc.emit('content-error', { error, editor }); }; -const onEditorExceptionCaught = ({ error, editor }) => { - proxy.$superdoc.emit('exception-caught', { error, editor }); +const onEditorException = ({ error, editor }) => { + proxy.$superdoc.emit('exception', { error, editor }); }; const updateToolbarState = () => { @@ -300,7 +300,7 @@ const editorOptions = (doc) => { onSelectionUpdate: onEditorSelectionChange, onCollaborationReady: onEditorCollaborationReady, onContentError: onEditorContentError, - onExceptionCaught: onEditorExceptionCaught, + onException: onEditorException, // onCommentsLoaded, // onCommentClicked, // onCommentsUpdate: onEditorCommentsUpdate, diff --git a/packages/superdoc/src/core/SuperDoc.js b/packages/superdoc/src/core/SuperDoc.js index cd94e698c5..3754fdc22e 100644 --- a/packages/superdoc/src/core/SuperDoc.js +++ b/packages/superdoc/src/core/SuperDoc.js @@ -72,7 +72,7 @@ export class SuperDoc extends EventEmitter { onPdfDocumentReady: () => null, onSidebarToggle: () => null, onCollaborationReady: () => null, - onExceptionCaught: () => null, + onException: () => null, // Image upload handler // async (file) => url; @@ -165,7 +165,7 @@ export class SuperDoc extends EventEmitter { this.on('sidebar-toggle', this.config.onSidebarToggle); this.on('collaboration-ready', this.config.onCollaborationReady); this.on('content-error', this.onContentError); - this.on('exception-caught', this.config.onExceptionCaught); + this.on('exception', this.config.onException); } /* ** @@ -445,6 +445,9 @@ export class SuperDoc extends EventEmitter { }); this.superdocStore.reset(); + + // Clean up telemetry when editor is destroyed + this.telemetry?.destroy(); this.app.unmount(); this.removeAllListeners();