From 9493a4df821d9369eff9ef8516a2a1e45f05c5fe Mon Sep 17 00:00:00 2001 From: Vladyslava Andrushchenko Date: Wed, 19 Feb 2025 18:54:05 +0200 Subject: [PATCH 1/4] Telemetry v1 updates --- .../v2/importer/docxImporter.js | 208 ++++-------- packages/superdoc/src/core/SuperDoc.js | 3 +- shared/common/Telemetry.js | 310 ++++++++++-------- 3 files changed, 248 insertions(+), 273 deletions(-) diff --git a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js index ab6eb5827a..c33e72e117 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js @@ -38,17 +38,46 @@ export const createDocumentJson = (docx, converter, editor) => { const json = carbonCopy(getInitialJSON(docx)); if (!json) return null; - const nodeListHandler = defaultNodeListHandler(); + // Track initial document structure + if (converter?.telemetry) { + const files = Object.keys(docx).map((filePath) => { + const parts = filePath.split('/'); + return { + filePath, + fileDepth: parts.length, + fileType: filePath.split('.').pop(), + }; + }); + converter.telemetry.trackFileStructure( + { + totalFiles: files.length, + maxDepth: Math.max(...files.map((f) => f.fileDepth)), + totalNodes: 0, + files, + }, + converter.fileSource, + converter.documentId, + converter.documentInternalId, + ); + } + + const nodeListHandler = defaultNodeListHandler(); const bodyNode = json.elements[0].elements.find((el) => el.name === 'w:body'); + if (bodyNode) { const node = bodyNode; const ignoreNodes = ['w:sectPr']; const content = node.elements?.filter((n) => !ignoreNodes.includes(n.name)) ?? []; const parsedContent = nodeListHandler.handler({ - nodes: content, nodeListHandler, docx, converter, editor + nodes: content, + nodeListHandler, + docx, + converter, + editor, }); + const result = { type: 'doc', content: parsedContent, @@ -57,19 +86,6 @@ export const createDocumentJson = (docx, converter, editor) => { }, }; - // Not empty document - if (result.content.length > 1) { - converter?.telemetry?.trackUsage( - converter?.fileSource, - converter?.documentId, - 'document_import', - { - documentType: 'docx', - internalId: converter?.documentInternalId, - timestamp: new Date().toISOString() - }); - } - return { pmDoc: result, savedTagsToRestore: node, @@ -114,7 +130,7 @@ const createNodeListHandler = (nodeHandlers) => { * @param {number} index Index to check * @returns {Object} Safe context object */ - const getSafeElementContext = (elements, index) => { + const getSafeElementContext = (elements, index, path) => { if (!elements || index < 0 || index >= elements.length) { return { elementIndex: index, @@ -125,11 +141,9 @@ const createNodeListHandler = (nodeHandlers) => { const element = elements[index]; return { - elementIndex: index, elementName: element?.name, - elementAttributes: element?.attributes, - hasElements: !!element?.elements, - elementCount: element?.elements?.length + attributes: element?.attributes, + elementPath: path, }; }; @@ -137,24 +151,12 @@ const createNodeListHandler = (nodeHandlers) => { if (!elements || !elements.length) return []; const processedElements = []; - const unhandledNodes = []; try { for (let index = 0; index < elements.length; index++) { try { const nodesToHandle = elements.slice(index); if (!nodesToHandle || nodesToHandle.length === 0) { - converter?.telemetry?.trackParsing( - converter?.fileSource, - converter?.documentId, - 'node', - 'empty_slice', - `/word/${filename || 'document.xml'}`, - { - index, - internalId: converter?.documentInternalId, - totalElements: elements.length, - }); continue; } @@ -175,140 +177,66 @@ const createNodeListHandler = (nodeHandlers) => { { nodes: [], consumed: 0 } ); - // Track unhandled nodes with safe context + // Only track unhandled nodes that should have been handled + const context = getSafeElementContext(elements, index, `/word/${filename || 'document.xml'}`); if (unhandled) { - const context = getSafeElementContext(elements, index); if (!context.elementName) continue; - - unhandledNodes.push({ - name: context.elementName, - attributes: context.elementAttributes - }); - - converter?.telemetry?.trackParsing( - converter?.fileSource, - converter?.documentId, - 'node', - 'unhandled', - `/word/${filename || 'document.xml'}`, - { - context, - internalId: converter?.documentInternalId, - }); + converter?.telemetry?.trackStatistic('unknown', context); continue; + } else { + converter?.telemetry?.trackStatistic('node', context); } - - // Bounds check before incrementing index - if (consumed > 0) { - index += consumed - 1; - if (index < 0) { - converter?.telemetry?.trackParsing( - converter?.fileSource, - converter?.documentId, - 'node', - 'invalid_index', - `/word/${filename || 'document.xml'}`, - { - originalIndex: index - (consumed - 1), - consumed, - resultingIndex: index, - internalId: converter?.documentInternalId, - } - ); - index = 0; // Reset to safe value - } - } - - // Process valid nodes - for (let node of (nodes || [])) { - if (node?.type) { - const ignore = ['runProperties']; - - if (node.type === 'text' && Array.isArray(node.content) && !node.content.length) { - - converter?.telemetry?.trackParsing( - converter?.fileSource, - converter?.documentId, - 'node', - 'empty_text', - `/word/${filename || 'document.xml'}`, - { - context: node.attrs, - internalId: converter?.documentInternalId, - }); - continue; - } - if (!ignore.includes(node.type)) { + // Process and store nodes (no tracking needed for success) + if (nodes) { + nodes.forEach((node) => { + if (node?.type && !['runProperties'].includes(node.type)) { + if (node.type === 'text' && Array.isArray(node.content) && !node.content.length) { + return; + } processedElements.push(node); } - } + }); } } catch (error) { editor?.emit('exception', { error }); - - const context = getSafeElementContext(elements, index); - if (error.details) { - context.elementAttributes = error.details; - } - - // Track individual element processing errors with safe context - converter?.telemetry?.trackParsing( - converter?.fileSource, - converter?.documentId, - 'element', - 'processing_error', - `/word/${filename || 'document.xml'}`, - { - error: { - message: error.message, - name: error.name, - stack: error.stack, - }, - internalId: converter?.documentInternalId, - context - } - ); + + converter?.telemetry?.trackStatistic('error', { + type: 'processing_error', + message: error.message, + name: error.name, + stack: error.stack, + fileName: `/word/${filename || 'document.xml'}`, + }); } } return processedElements; } catch (error) { editor?.emit('exception', { error }); + + // Track only catastrophic handler failures + converter?.telemetry?.trackStatistic('error', { + type: 'fatal_error', + message: error.message, + name: error.name, + stack: error.stack, + fileName: `/word/${filename || 'document.xml'}`, + }); - // Track catastrophic errors in the node list handler - converter?.telemetry?.trackParsing( - converter?.fileSource, - converter?.documentId, - 'handler', - 'nodeListHandler', - `/word/${filename || 'document.xml'}`, - { - status: 'error', - error: { - message: error.message, - name: error.name, - stack: error.stack - }, - internalId: converter?.documentInternalId, - context: { - totalElements: elements?.length || 0, - processedCount: processedElements.length, - unhandledCount: unhandledNodes.length, - } - } - ); throw error; } }; - return nodeListHandlerFn; }; /** * * @param {XmlNode} node + * @param {ParsedDocx} docx + * @param {SuperConverter} converter instance. + * @param {Editor} editor instance. * @returns {Object} The document styles object */ function getDocumentStyles(node, docx, converter, editor) { @@ -397,4 +325,4 @@ function getHeaderFooter(el, elementType, docx, converter, editor) { storage[rId] = { type: 'doc', content: [...schema] }; storageIds[sectionType] = rId; -}; +} diff --git a/packages/superdoc/src/core/SuperDoc.js b/packages/superdoc/src/core/SuperDoc.js index 763338bfda..40191d680d 100644 --- a/packages/superdoc/src/core/SuperDoc.js +++ b/packages/superdoc/src/core/SuperDoc.js @@ -220,9 +220,10 @@ export class SuperDoc extends EventEmitter { #initTelemetry() { this.telemetry = new Telemetry({ enabled: this.config.telemetry?.enabled ?? true, - licenceKey: this.config.telemetry?.licenceKey, + licenseKey: this.config.telemetry?.licenseKey, endpoint: this.config.telemetry?.endpoint, superdocId: this.superdocId, + superdocVersion: this.version, }); } diff --git a/shared/common/Telemetry.js b/shared/common/Telemetry.js index 1e1e803bc1..2f11b08a46 100644 --- a/shared/common/Telemetry.js +++ b/shared/common/Telemetry.js @@ -1,43 +1,10 @@ -/** - * @typedef {Object} BaseEvent - * @property {string} id - Unique event identifier - * @property {string} timestamp - ISO timestamp of the event - * @property {string} sessionId - Current session identifier - * @property {string} superdocId - SuperDoc ID - * @property {Object} document - Document information - * @property {string} [document.id] - Reference ID - * @property {string} [document.type] - Document type - * @property {string} [document.internalId] - Internal document ID - * @property {string} [document.hash] - Document CRC32 hash - * @property {string} [document.lastModified] - Last modified timestamp - */ - -/** - * @typedef {Object} UsageEvent - * @param {File} fileSource - File object - * @param {string} documentId - document id - * @property {string} name - Name of the usage event - * @property {Object.} properties - Event properties - */ - -/** - * @typedef {Object} ParsingEvent - * @param {File} fileSource - File object - * @param {string} documentId - document id - * @property {'mark'|'element'} category - Category of the parsed item - * @property {string} name - Name of the parsed item - * @property {string} path - Document path where item was found - * @property {Object.} [metadata] - Additional context - */ - -/** @typedef {(UsageEvent & BaseEvent) | (ParsingEvent & BaseEvent)} TelemetryEvent */ - /** * @typedef {Object} TelemetryConfig - * @property {string} [licenceKey] - Licence key for telemetry service + * @property {string} [licenseKey] - License key for telemetry service * @property {boolean} [enabled=true] - Whether telemetry is enabled * @property {string} endpoint - service endpoint * @property {string} superdocId - SuperDoc id + * @property {string} superdocVersion - SuperDoc version */ import crc32 from 'buffer-crc32'; @@ -49,6 +16,9 @@ class Telemetry { /** @type {string} */ superdocId; + + /** @type {string} */ + superdocVersion; /** @type {string} */ licenseKey; @@ -59,17 +29,36 @@ class Telemetry { /** @type {string} */ sessionId; - /** @type {TelemetryEvent[]} */ - events = []; + /** @type {Object} */ + statistics = { + nodeTypes: {}, + markTypes: {}, + styleTypes: {}, + errorCount: 0, + }; + + /** @type {Array} */ + unknownElements = []; + + /** @type {Array} */ + errors = []; + + /** @type {Object} */ + fileStructure = { + totalFiles: 0, + maxDepth: 0, + totalNodes: 0, + files: [], + }; + + /** @type {Object} */ + documentInfo = null; /** @type {number|undefined} */ flushInterval; /** @type {number} */ - static BATCH_SIZE = 50; - - /** @type {number} */ - static FLUSH_INTERVAL = 10000; // 30 seconds + static FLUSH_INTERVAL = 10000; // 10 seconds /** @type {string} */ static COMMUNITY_LICENSE_KEY = 'community-and-eval-agplv3'; @@ -84,10 +73,11 @@ class Telemetry { constructor(config = {}) { this.enabled = config.enabled ?? true; - this.licenseKey = config.licenceKey ?? Telemetry.COMMUNITY_LICENSE_KEY; + this.licenseKey = config.licenseKey ?? Telemetry.COMMUNITY_LICENSE_KEY; this.endpoint = config.endpoint ?? Telemetry.DEFAULT_ENDPOINT; this.superdocId = config.superdocId; + this.superdocVersion = config.superdocVersion; this.sessionId = this.generateId(); if (this.enabled) { @@ -96,15 +86,16 @@ class Telemetry { } /** - * Create source payload for request + * Get browser environment information + * @returns {Object} Browser information */ - getSourceData() { + getBrowserInfo() { return { userAgent: window.navigator.userAgent, - url: window.location.href, - host: window.location.host, - referrer: document.referrer, - screen: { + currentUrl: window.location.href, + hostname: window.location.hostname, + referrerUrl: document.referrer, + screenSize: { width: window.screen.width, height: window.screen.height, }, @@ -112,101 +103,115 @@ class Telemetry { } /** - * Track feature usage - * @param {File} fileSource - File object - * @param {string} documentId - document id - * @param {string} name - Name of the feature/event - * @param {Object.} [properties] - Additional properties + * Track document usage event + * @param {File} fileSource - Document file + * @param {string} documentId - Document identifier + * @param {string} name - Event name + * @param {Object} properties - Additional properties */ async trackUsage(fileSource, documentId, name, properties = {}) { + // ToDo rewrite when decide how it has to look like + // For example to call endpoint every time usage event is fired if (!this.enabled) return; - if (!fileSource) { - console.warn('Telemetry: missing file source'); - return; - } - - const processedDoc = await this.processDocument(fileSource, { + const docInfo = await this.processDocument(fileSource, { id: documentId, internalId: properties.internalId, }); - /** @type {UsageEvent & BaseEvent} */ const event = { id: this.generateId(), type: 'usage', timestamp: new Date().toISOString(), sessionId: this.sessionId, superdocId: this.superdocId, - source: this.getSourceData(), - document: processedDoc, + superdocVersion: this.superdocVersion, + document: docInfo, + browser: this.getBrowserInfo(), name, properties, }; - - this.queueEvent(event); } /** - * Track parsing events - * @param {File} fileSource - File object - * @param {string} documentId - document id - * @param {'mark'|'element'} category - Category of parsed item - * @param {string} name - Name of the item - * @param {string} path - Document path where item was found - * @param {Object.} [metadata] - Additional context + * Track parsing statistics + * @param {string} category - Statistic category + * @param {string|Object} data - Statistic data */ - async trackParsing(fileSource, documentId, category, name, path, metadata) { - if (!this.enabled) return; - - if (!fileSource) { - console.warn('Telemetry: missing file source'); - return; + trackStatistic(category, data) { + if (category === 'node') { + this.statistics.nodeTypes[data.elementName] = (this.statistics.nodeTypes[data.elementName] || 0) + 1; + this.fileStructure.totalNodes++; + } else if (category === 'mark') { + this.statistics.markTypes[data] = (this.statistics.markTypes[data] || 0) + 1; + } else if (category === 'style') { + const { type, value } = data; + if (!this.statistics.styleTypes[type]) { + this.statistics.styleTypes[type] = {}; + } + this.statistics.styleTypes[type][value] = (this.statistics.styleTypes[type][value] || 0) + 1; + } else if (category === 'unknown') { + const addedElement = this.unknownElements.find(e => e.elementName === data.elementName); + if (addedElement) { + addedElement.count += 1; + addedElement.attributes = { + ...addedElement.attributes, + ...data.attributes + }; + } else { + this.unknownElements.push({ + ...data, + count: 1 + }); + } + } else if (category === 'error') { + this.errors.push(data); + this.statistics.errorCount++; } + } - const processedDoc = await this.processDocument(fileSource, { + /** + * Track file structure + * @param {Object} structure - File structure information + * @param {File} fileSource - original file + * @param {String} documentId - document ID + * @param {string} internalId - document ID form settings.xml + */ + async trackFileStructure(structure, fileSource, documentId, internalId) { + this.fileStructure = structure; + this.documentInfo = await this.processDocument(fileSource, { id: documentId, - internalId: metadata.internalId, + internalId: internalId, }); - - /** @type {ParsingEvent & BaseEvent} */ - const event = { - id: this.generateId(), - type: 'parsing', - timestamp: new Date().toISOString(), - sessionId: this.sessionId, - superdocId: this.superdocId, - source: this.getSourceData(), - category, - document: processedDoc, - name, - path, - ...(metadata && { metadata }), - }; - - this.queueEvent(event); } /** * Process document metadata * @param {File} file - Document file - * @param {Object} options - Additional metadata options + * @param {Object} options - Additional options * @returns {Promise} Document metadata */ async processDocument(file, options = {}) { + if (!file) { + console.warn('Telemetry: missing file source'); + return {}; + } + let hash = ''; try { hash = await this.generateCrc32Hash(file); } catch (error) { - console.error('Failed to retrieve file hash:', error); + console.error('Failed to generate file hash:', error); } - + return { id: options.id, - type: options.type || file.type, - internalId: options.internalId, - hash, + name: file.name, + size: file.size, + crc32: hash, lastModified: file.lastModified ? new Date(file.lastModified).toISOString() : null, + type: file.type || 'docx', + internalId: options.internalId, }; } @@ -220,32 +225,50 @@ class Telemetry { const arrayBuffer = await file.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); const hashBuffer = crc32(buffer); - const hashArray = Array.from(hashBuffer); - return hashArray.map((b) => b.toString(16).padStart(2, '0')).join(''); + return hashBuffer.toString('hex'); } - - /** - * Queue event for sending - * @param {TelemetryEvent} event - * @private - */ - queueEvent(event) { - this.events.push(event); - - if (this.events.length >= Telemetry.BATCH_SIZE) { - this.flush(); + + isTelemetryDataChanged() { + const initialStatistics = { + nodeTypes: {}, + markTypes: {}, + styleTypes: {}, + unknownElements: [], + errorCount: 0, + }; + const initialFileStructure = { + totalFiles: 0, + maxDepth: 0, + totalNodes: 0, + files: [], } + // Empty document case + if (Object.keys(this.statistics.nodeTypes).length <= 1) return; + + return this.statistics !== initialStatistics || initialFileStructure !== this.fileStructure || this.errors.length > 0 || this.unknownElements.length > 0; } /** - * Flush queued events to server + * Sends current report * @returns {Promise} */ - async flush() { - if (!this.enabled || !this.events.length) return; - - const eventsToSend = [...this.events]; - this.events = []; + async sendReport() { + if (!this.enabled || !this.isTelemetryDataChanged()) return; + + const report = { + id: this.generateId(), + type: 'parsing', + timestamp: new Date().toISOString(), + sessionId: this.sessionId, + superdocId: this.superdocId, + superdocVersion: this.superdocVersion, + file: this.documentInfo, + browser: this.getBrowserInfo(), + statistics: this.statistics, + fileStructure: this.fileStructure, + unknownElements: this.unknownElements, + errors: this.errors, + } try { const response = await fetch(this.endpoint, { @@ -254,16 +277,16 @@ class Telemetry { 'Content-Type': 'application/json', 'X-License-Key': this.licenseKey, }, - body: JSON.stringify(eventsToSend), + body: JSON.stringify(report), }); if (!response.ok) { throw new Error(`Upload failed: ${response.statusText}`); + } else { + this.resetStatistics(); } } catch (error) { console.error('Failed to upload telemetry:', error); - // Add events back to queue - this.events = [...eventsToSend, ...this.events]; } } @@ -273,31 +296,54 @@ class Telemetry { */ startPeriodicFlush() { this.flushInterval = setInterval(() => { - if (this.events.length > 0) { - this.flush(); - } + this.sendReport(); }, Telemetry.FLUSH_INTERVAL); } /** * Generate unique identifier - * @returns {string} + * @returns {string} Unique ID * @private */ generateId() { - const randomValue = randomBytes(4).toString('hex'); - return `${Date.now()}-${randomValue}`; + const timestamp = Date.now(); + const random = randomBytes(4).toString('hex'); + return `${timestamp}-${random}`; + } + + /** + * Reset statistics + */ + resetStatistics() { + this.statistics = { + nodeTypes: {}, + markTypes: {}, + styleTypes: {}, + unknownElements: [], + errorCount: 0, + }; + + this.fileStructure = { + totalFiles: 0, + maxDepth: 0, + totalNodes: 0, + files: [], + }; + + this.unknownElements = []; + + this.errors = []; } /** * Clean up telemetry service * @returns {Promise} */ - destroy() { + async destroy() { if (this.flushInterval) { clearInterval(this.flushInterval); } - return this.flush(); + return this.sendReport(); } } From 3754ba9e508409e90db540f89a5c4bde1e4d914e Mon Sep 17 00:00:00 2001 From: Vladyslava Andrushchenko Date: Thu, 20 Feb 2025 18:07:12 +0200 Subject: [PATCH 2/4] track marks and attributes --- .../core/super-converter/SuperConverter.js | 4 ++ .../v2/importer/docxImporter.js | 24 +++++-- shared/common/Telemetry.js | 70 ++++++++----------- 3 files changed, 54 insertions(+), 44 deletions(-) diff --git a/packages/super-editor/src/core/super-converter/SuperConverter.js b/packages/super-editor/src/core/super-converter/SuperConverter.js index d09788a1e3..7e96289555 100644 --- a/packages/super-editor/src/core/super-converter/SuperConverter.js +++ b/packages/super-editor/src/core/super-converter/SuperConverter.js @@ -271,6 +271,10 @@ class SuperConverter { getSchema(editor) { this.getDocumentInternalId(); const result = createDocumentJson({...this.convertedXml, media: this.media }, this, editor); + + setTimeout(() => { + this.telemetry?.sendReport(); + }, 6000); if (result) { this.savedTagsToRestore.push({ ...result.savedTagsToRestore }); diff --git a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js index c33e72e117..1dcee06183 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js @@ -128,9 +128,11 @@ const createNodeListHandler = (nodeHandlers) => { * Gets safe element context even if index is out of bounds * @param {Array} elements Array of elements * @param {number} index Index to check + * @param {Object} processedNode result node + * @param {String} path Occurrence filename * @returns {Object} Safe context object */ - const getSafeElementContext = (elements, index, path) => { + const getSafeElementContext = (elements, index, processedNode, path) => { if (!elements || index < 0 || index >= elements.length) { return { elementIndex: index, @@ -142,8 +144,11 @@ const createNodeListHandler = (nodeHandlers) => { const element = elements[index]; return { elementName: element?.name, - attributes: element?.attributes, + attributes: processedNode?.attrs, + marks: processedNode?.marks, elementPath: path, + type: processedNode?.type, + content: processedNode?.content, }; }; @@ -178,14 +183,25 @@ const createNodeListHandler = (nodeHandlers) => { ); // Only track unhandled nodes that should have been handled - const context = getSafeElementContext(elements, index, `/word/${filename || 'document.xml'}`); + const context = getSafeElementContext(elements, index, nodes[0], `/word/${filename || 'document.xml'}`); if (unhandled) { if (!context.elementName) continue; - converter?.telemetry?.trackStatistic('unknown', context); + const ignoreElements = ['w:pPr', 'w:tcPr']; + if (!ignoreElements.includes(context.elementName)) { + converter?.telemetry?.trackStatistic('unknown', context); + } continue; } else { converter?.telemetry?.trackStatistic('node', context); + + // Use Telemetry to track list item attributes + if (context.type === 'orderedList' || context.type === 'bulletList') { + context.content.forEach((item) => { + const innerItemContext = getSafeElementContext([item], 0, item, `/word/${filename || 'document.xml'}`); + converter?.telemetry?.trackStatistic('attributes', innerItemContext); + }) + } } // Process and store nodes (no tracking needed for success) diff --git a/shared/common/Telemetry.js b/shared/common/Telemetry.js index 2f11b08a46..592b1550f9 100644 --- a/shared/common/Telemetry.js +++ b/shared/common/Telemetry.js @@ -33,7 +33,7 @@ class Telemetry { statistics = { nodeTypes: {}, markTypes: {}, - styleTypes: {}, + attributes: {}, errorCount: 0, }; @@ -54,12 +54,6 @@ class Telemetry { /** @type {Object} */ documentInfo = null; - /** @type {number|undefined} */ - flushInterval; - - /** @type {number} */ - static FLUSH_INTERVAL = 10000; // 10 seconds - /** @type {string} */ static COMMUNITY_LICENSE_KEY = 'community-and-eval-agplv3'; @@ -79,10 +73,6 @@ class Telemetry { this.superdocVersion = config.superdocVersion; this.sessionId = this.generateId(); - - if (this.enabled) { - this.startPeriodicFlush(); - } } /** @@ -142,14 +132,6 @@ class Telemetry { if (category === 'node') { this.statistics.nodeTypes[data.elementName] = (this.statistics.nodeTypes[data.elementName] || 0) + 1; this.fileStructure.totalNodes++; - } else if (category === 'mark') { - this.statistics.markTypes[data] = (this.statistics.markTypes[data] || 0) + 1; - } else if (category === 'style') { - const { type, value } = data; - if (!this.statistics.styleTypes[type]) { - this.statistics.styleTypes[type] = {}; - } - this.statistics.styleTypes[type][value] = (this.statistics.styleTypes[type][value] || 0) + 1; } else if (category === 'unknown') { const addedElement = this.unknownElements.find(e => e.elementName === data.elementName); if (addedElement) { @@ -168,6 +150,35 @@ class Telemetry { this.errors.push(data); this.statistics.errorCount++; } + + if (data.marks?.length) { + data.marks.forEach((mark) => { + this.statistics.markTypes[mark.type] = (this.statistics.markTypes[mark.type] || 0) + 1; + }); + } + + // Style attributes + if (data.attributes && Object.keys(data.attributes).length) { + const styleAttributes = [ + 'textIndent', + 'textAlign', + 'spacing', + 'lineHeight', + 'indent', + 'list-style-type', + 'listLevel', + 'textStyle', + 'order', + 'lvlText', + 'lvlJc', + 'listNumberingType', + 'numId' + ]; + Object.keys(data.attributes).forEach((attribute) => { + if (!styleAttributes.includes(attribute)) return; + this.statistics.attributes[attribute] = (this.statistics.attributes[attribute] || 0) + 1; + }); + } } /** @@ -289,16 +300,6 @@ class Telemetry { console.error('Failed to upload telemetry:', error); } } - - /** - * Start periodic flush interval - * @private - */ - startPeriodicFlush() { - this.flushInterval = setInterval(() => { - this.sendReport(); - }, Telemetry.FLUSH_INTERVAL); - } /** * Generate unique identifier @@ -334,17 +335,6 @@ class Telemetry { this.errors = []; } - - /** - * Clean up telemetry service - * @returns {Promise} - */ - async destroy() { - if (this.flushInterval) { - clearInterval(this.flushInterval); - } - return this.sendReport(); - } } export { Telemetry }; From e14d2fdbe3b7e45575a09feddc6b13eadff4e8df Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Thu, 20 Feb 2025 16:21:24 -0300 Subject: [PATCH 3/4] refactor: Telemetry report structure and minor formatting --- package-lock.json | 2 +- shared/common/Telemetry.js | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index 637204d423..f85a8f6ab8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10270,7 +10270,7 @@ }, "packages/superdoc": { "name": "@harbour-enterprises/superdoc", - "version": "0.4.37-RC1", + "version": "0.4.40", "license": "AGPL-3.0", "dependencies": { "@harbour-enterprises/super-editor": "0.0.1-alpha.0", diff --git a/shared/common/Telemetry.js b/shared/common/Telemetry.js index 592b1550f9..32b0c01662 100644 --- a/shared/common/Telemetry.js +++ b/shared/common/Telemetry.js @@ -16,7 +16,7 @@ class Telemetry { /** @type {string} */ superdocId; - + /** @type {string} */ superdocVersion; @@ -150,13 +150,13 @@ class Telemetry { this.errors.push(data); this.statistics.errorCount++; } - + if (data.marks?.length) { data.marks.forEach((mark) => { this.statistics.markTypes[mark.type] = (this.statistics.markTypes[mark.type] || 0) + 1; }); } - + // Style attributes if (data.attributes && Object.keys(data.attributes).length) { const styleAttributes = [ @@ -238,7 +238,7 @@ class Telemetry { const hashBuffer = crc32(buffer); return hashBuffer.toString('hex'); } - + isTelemetryDataChanged() { const initialStatistics = { nodeTypes: {}, @@ -255,7 +255,7 @@ class Telemetry { } // Empty document case if (Object.keys(this.statistics.nodeTypes).length <= 1) return; - + return this.statistics !== initialStatistics || initialFileStructure !== this.fileStructure || this.errors.length > 0 || this.unknownElements.length > 0; } @@ -265,8 +265,8 @@ class Telemetry { */ async sendReport() { if (!this.enabled || !this.isTelemetryDataChanged()) return; - - const report = { + + const report = [{ id: this.generateId(), type: 'parsing', timestamp: new Date().toISOString(), @@ -279,8 +279,8 @@ class Telemetry { fileStructure: this.fileStructure, unknownElements: this.unknownElements, errors: this.errors, - } - + }]; + try { const response = await fetch(this.endpoint, { method: 'POST', @@ -300,7 +300,7 @@ class Telemetry { console.error('Failed to upload telemetry:', error); } } - + /** * Generate unique identifier * @returns {string} Unique ID @@ -330,9 +330,9 @@ class Telemetry { totalNodes: 0, files: [], }; - + this.unknownElements = []; - + this.errors = []; } } From db92f49afd6e232ef848b6249fb00711c5abb723 Mon Sep 17 00:00:00 2001 From: Vladyslava Andrushchenko Date: Fri, 21 Feb 2025 13:41:23 +0200 Subject: [PATCH 4/4] add usage event for import and minor fixes --- packages/super-editor/src/core/Editor.js | 4 ++- .../core/super-converter/SuperConverter.js | 5 +--- .../v2/importer/docxImporter.js | 18 +++++++++---- .../v2/importer/importerHelpers.js | 4 +++ .../v2/importer/standardNodeImporter.js | 21 ++++++++++++--- shared/common/Telemetry.js | 26 ++++++++++--------- 6 files changed, 52 insertions(+), 26 deletions(-) diff --git a/packages/super-editor/src/core/Editor.js b/packages/super-editor/src/core/Editor.js index 0db05e84de..b29602baa5 100644 --- a/packages/super-editor/src/core/Editor.js +++ b/packages/super-editor/src/core/Editor.js @@ -567,6 +567,8 @@ export class Editor extends EventEmitter { const dom = this.view.dom; dom.editor = this; + + this.options.telemetry?.sendReport(); } /** @@ -912,7 +914,7 @@ export class Editor extends EventEmitter { isHeadless: this.options.isHeadless, }); - this.telemetry?.trackUsage('document_export', { + this.options.telemetry?.trackUsage('document_export', { documentType: 'docx', timestamp: new Date().toISOString() }); diff --git a/packages/super-editor/src/core/super-converter/SuperConverter.js b/packages/super-editor/src/core/super-converter/SuperConverter.js index 7e96289555..09834d68ad 100644 --- a/packages/super-editor/src/core/super-converter/SuperConverter.js +++ b/packages/super-editor/src/core/super-converter/SuperConverter.js @@ -55,6 +55,7 @@ class SuperConverter { 'w:rPr': 'runProperties', 'w:sectPr': 'sectionProperties', 'w:numPr': 'numberingProperties', + 'w:tcPr': 'tableCellProperties', }); static elements = new Set(['w:document', 'w:body', 'w:p', 'w:r', 'w:t', 'w:delText']); @@ -271,10 +272,6 @@ class SuperConverter { getSchema(editor) { this.getDocumentInternalId(); const result = createDocumentJson({...this.convertedXml, media: this.media }, this, editor); - - setTimeout(() => { - this.telemetry?.sendReport(); - }, 6000); if (result) { this.savedTagsToRestore.push({ ...result.savedTagsToRestore }); diff --git a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js index 1dcee06183..e8fb9d6ee0 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/docxImporter.js @@ -85,6 +85,17 @@ export const createDocumentJson = (docx, converter, editor) => { attributes: json.elements[0].attributes, }, }; + + // Not empty document + if (result.content.length > 1) { + converter?.telemetry?.trackUsage( + 'document_import', + { + documentType: 'docx', + timestamp: new Date().toISOString() + } + ); + } return { pmDoc: result, @@ -186,11 +197,8 @@ const createNodeListHandler = (nodeHandlers) => { const context = getSafeElementContext(elements, index, nodes[0], `/word/${filename || 'document.xml'}`); if (unhandled) { if (!context.elementName) continue; - - const ignoreElements = ['w:pPr', 'w:tcPr']; - if (!ignoreElements.includes(context.elementName)) { - converter?.telemetry?.trackStatistic('unknown', context); - } + + converter?.telemetry?.trackStatistic('unknown', context); continue; } else { converter?.telemetry?.trackStatistic('node', context); diff --git a/packages/super-editor/src/core/super-converter/v2/importer/importerHelpers.js b/packages/super-editor/src/core/super-converter/v2/importer/importerHelpers.js index 0f3d6bebc0..a3ffc7f955 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/importerHelpers.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/importerHelpers.js @@ -91,6 +91,10 @@ export function getElementName(element) { return SuperConverter.allowedElements[element.name || element.type]; } +export const isPropertiesElement = (element) => { + return !!SuperConverter.propertyTypes[element.name || element.type]; +} + /** * * @param {XmlNode[]} elements diff --git a/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js b/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js index 1075e03234..0867eed9e0 100644 --- a/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js +++ b/packages/super-editor/src/core/super-converter/v2/importer/standardNodeImporter.js @@ -1,4 +1,4 @@ -import { getElementName, parseProperties } from './importerHelpers.js'; +import { getElementName, parseProperties, isPropertiesElement } from './importerHelpers.js'; /** * @type {import("docxImporter").NodeHandler} @@ -14,6 +14,19 @@ export const handleStandardNode = (params) => { const { name } = node; const { attributes, elements, marks = [] } = parseProperties(node, docx); + // Formatting only nodes + if (isPropertiesElement(node)) { + return { + nodes: [{ + type: getElementName(node), + attrs: { ...attributes }, + marks: [], + }], + consumed: 0 + }; + } + + // Unhandled nodes if (!getElementName(node)) { return { nodes: [{ @@ -30,7 +43,7 @@ export const handleStandardNode = (params) => { // Iterate through the children and build the schemaNode content // Skip run properties since they are formatting only elements const content = []; - if (elements && elements.length && name !== 'w:rPr') { + if (elements && elements.length) { const updatedElements = elements.map((el) => { if (!el.marks) el.marks = []; el.marks.push(...marks); @@ -41,14 +54,14 @@ export const handleStandardNode = (params) => { const childContent = nodeListHandler.handler(childParams); content.push(...childContent); } - + const resultNode = { type: getElementName(node), content, attrs: { ...attributes }, marks: [], }; - + return { nodes: [resultNode], consumed: 1 }; }; diff --git a/shared/common/Telemetry.js b/shared/common/Telemetry.js index 32b0c01662..41e3efc5de 100644 --- a/shared/common/Telemetry.js +++ b/shared/common/Telemetry.js @@ -94,21 +94,12 @@ class Telemetry { /** * Track document usage event - * @param {File} fileSource - Document file - * @param {string} documentId - Document identifier * @param {string} name - Event name * @param {Object} properties - Additional properties */ - async trackUsage(fileSource, documentId, name, properties = {}) { - // ToDo rewrite when decide how it has to look like - // For example to call endpoint every time usage event is fired + async trackUsage(name, properties = {}) { if (!this.enabled) return; - const docInfo = await this.processDocument(fileSource, { - id: documentId, - internalId: properties.internalId, - }); - const event = { id: this.generateId(), type: 'usage', @@ -116,11 +107,13 @@ class Telemetry { sessionId: this.sessionId, superdocId: this.superdocId, superdocVersion: this.superdocVersion, - document: docInfo, + file: this.documentInfo, browser: this.getBrowserInfo(), name, properties, }; + + await this.sendDataToTelemetry(event); } /** @@ -280,7 +273,16 @@ class Telemetry { unknownElements: this.unknownElements, errors: this.errors, }]; + + await this.sendDataToTelemetry(report); + } + /** + * Sends data to the service + * @returns {Object} payload + * @returns {Promise} + */ + async sendDataToTelemetry(data) { try { const response = await fetch(this.endpoint, { method: 'POST', @@ -288,7 +290,7 @@ class Telemetry { 'Content-Type': 'application/json', 'X-License-Key': this.licenseKey, }, - body: JSON.stringify(report), + body: JSON.stringify(data), }); if (!response.ok) {