From 2698f5dd71dc0c72a326f306094aab5a40f1c7c4 Mon Sep 17 00:00:00 2001 From: Uria Rogel JMT Date: Tue, 25 Apr 2023 18:06:35 +0200 Subject: [PATCH 1/8] works in node and browser --- seq_logger.js | 133 ++++++++++++++++++-------------------------------- 1 file changed, 47 insertions(+), 86 deletions(-) diff --git a/seq_logger.js b/seq_logger.js index ec09b1b..2227003 100644 --- a/seq_logger.js +++ b/seq_logger.js @@ -1,18 +1,15 @@ "use strict"; -let http = require('http'); -let https = require('https'); -let url = require('url'); - +const NodeBlob = require('buffer').Blob +const GlobalBlob = Blob !== undefined ? Blob : NodeBlob const HEADER = '{"Events":['; const FOOTER = "]}"; -const HEADER_FOOTER_BYTES = Buffer.byteLength(HEADER, 'utf8') + Buffer.byteLength(FOOTER, 'utf8'); - +const HEADER_FOOTER_BYTES = (new GlobalBlob([HEADER])).size + (new GlobalBlob([FOOTER])).size class SeqLogger { constructor(config) { let dflt = { - serverUrl: 'http://localhost:5341', + serverUrl: 'http://seq:5341', apiKey: null, maxBatchingTime: 2000, eventSizeLimit: 256 * 1024, @@ -27,7 +24,7 @@ class SeqLogger { if (!serverUrl.endsWith('/')) { serverUrl += '/'; } - this._endpoint = url.parse(serverUrl + 'api/events/raw'); + this._endpoint = serverUrl + 'api/events/raw'; this._apiKey = cfg.apiKey || dflt.apiKey; this._maxBatchingTime = cfg.maxBatchingTime || dflt.maxBatchingTime; this._eventSizeLimit = cfg.eventSizeLimit || dflt.eventSizeLimit; @@ -43,12 +40,6 @@ class SeqLogger { this._activeShipper = null; this._onRemoteConfigChange = cfg.onRemoteConfigChange || null; this._lastRemoteConfig = null; - - this._httpModule = this._endpoint.protocol === "https:" ? https : http - this._httpAgent = new this._httpModule.Agent({ - keepAlive: true, - maxTotalSockets: 25, // recommendation from https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/node-configuring-maxsockets.html - }); } /** @@ -250,12 +241,12 @@ class SeqLogger { messageTemplate: "[seq] Circular structure found" }); } - var jsonLen = Buffer.byteLength(json, 'utf8'); + var jsonLen = new GlobalBlob([json]).size; if (jsonLen > this._eventSizeLimit) { this._onError("[seq] Event body is larger than " + this._eventSizeLimit + " bytes: " + json); this._queue[i] = next = this._eventTooLargeErrorEvent(next); json = JSON.stringify(next); - jsonLen = Buffer.byteLength(json, 'utf8'); + jsonLen = new GlobalBlob([json]).size; } // Always try to send a batch of at least one event, even if the batch size is @@ -285,77 +276,48 @@ class SeqLogger { return new Promise((resolve, reject) => { const sendRequest = (batch, bytes) => { attempts++; - let req = this._httpModule.request({ - host: this._endpoint.hostname, - port: this._endpoint.port, - path: this._endpoint.path, - protocol: this._endpoint.protocol, - agent: this._httpAgent, - headers: { - "Content-Type": "application/json", - "X-Seq-ApiKey": this._apiKey ? this._apiKey : null, - "Content-Length": bytes, - }, - method: "POST", - timeout: this._requestTimeout - }); - - req.on("socket", (socket) => { - if (socket.listeners("timeout").length == 0) { - socket.on("timeout", () => { - req.destroy(); - if (attempts > this._maxRetries) { - return reject('HTTP log shipping failed, reached timeout (' + this._requestTimeout + ' ms)') - } else { - return setTimeout(() => sendRequest(batch, bytes), this._retryDelay); - } - }); - } - }); - - req.on('response', res => { - var httpErr = null; - if (res.statusCode !== 200 && res.statusCode !== 201) { + // TODO: add `timeout` with `AbortController` and `signal` + fetch(this._endpoint, { + keepalive: true, + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Seq-ApiKey": this._apiKey ? this._apiKey : null, + "Content-Length": bytes, + }, + body: `${HEADER}${batch.join(',')}${FOOTER}` + }) + .then((res) => { + let httpErr = null; + if (res.status !== 200 && res.status !== 201) { httpErr = 'HTTP log shipping failed: ' + res.statusCode; } - - res.on('data', (buffer) => { - let dataRaw = buffer.toString(); - - if (this._onRemoteConfigChange && this._lastRemoteConfig !== dataRaw) { - this._lastRemoteConfig = dataRaw; - this._onRemoteConfigChange(JSON.parse(dataRaw)); - } - }); - - res.on('error', e => { - return reject(e); - }); - res.on('end', () => { - if (httpErr !== null) { - if (this._httpOrNetworkError(res) && attempts < this._maxRetries) { - return setTimeout(() => sendRequest(batch, bytes), this._retryDelay); - } - return reject(httpErr); - } else { - return resolve(true); + if (httpErr !== null) { + if (this._httpOrNetworkError(res) && attempts < this._maxRetries) { + return setTimeout(() => sendRequest(batch, bytes), this._retryDelay); } - }); - }); - - req.on('error', e => { - return reject(e); - }); - - req.write(HEADER); - var delim = ""; - for (var b = 0; b < batch.length; b++) { - req.write(delim); - delim = ","; - req.write(batch[b]); - } - req.write(FOOTER); - req.end(); + return reject(httpErr); + } else { + return resolve(true); + } + }) + .catch((err) => { + console.log('logger error', err); + reject(err); + }) + // TODO: not migrated yet + // req.on("socket", (socket) => { + // if (socket.listeners("timeout").length == 0) { + // socket.on("timeout", () => { + // req.destroy(); + // if (attempts > this._maxRetries) { + // return reject('HTTP log shipping failed, reached timeout (' + this._requestTimeout + ' ms)') + // } else { + // return setTimeout(() => sendRequest(batch, bytes), this._retryDelay); + // } + // }); + // } + // }); } return sendRequest(batch, bytes); @@ -369,8 +331,7 @@ class SeqLogger { // CORS-safelisted for the Content-Type request header const options = { type: 'text/plain' }; - - const endpointWithKey = Object.assign({}, this._endpoint, { query: { 'apiKey': this._apiKey } }); + const endpointWithKey = Object.assign({}, new URL(this._endpoint), { query: { 'apiKey': this._apiKey } }); return { dataParts, From 796fef1b48a51686befa9543938067d1070056fb Mon Sep 17 00:00:00 2001 From: Uria Rogel JMT Date: Wed, 26 Apr 2023 10:08:13 +0200 Subject: [PATCH 2/8] implemented timeout with retry --- seq_logger.js | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/seq_logger.js b/seq_logger.js index 2227003..f14d82c 100644 --- a/seq_logger.js +++ b/seq_logger.js @@ -9,7 +9,7 @@ const HEADER_FOOTER_BYTES = (new GlobalBlob([HEADER])).size + (new GlobalBlob([F class SeqLogger { constructor(config) { let dflt = { - serverUrl: 'http://seq:5341', + serverUrl: 'http://localhost:5341', apiKey: null, maxBatchingTime: 2000, eventSizeLimit: 256 * 1024, @@ -275,8 +275,17 @@ class SeqLogger { return new Promise((resolve, reject) => { const sendRequest = (batch, bytes) => { + const controller = new AbortController() attempts++; - // TODO: add `timeout` with `AbortController` and `signal` + const timerId = setTimeout(() => { + controller.abort() + if (attempts > this._maxRetries) { + reject('HTTP log shipping failed, reached timeout (' + this._requestTimeout + ' ms)') + } else { + setTimeout(() => sendRequest(batch, bytes), this._retryDelay); + } + }, this._requestTimeout) + fetch(this._endpoint, { keepalive: true, method: "POST", @@ -285,9 +294,11 @@ class SeqLogger { "X-Seq-ApiKey": this._apiKey ? this._apiKey : null, "Content-Length": bytes, }, - body: `${HEADER}${batch.join(',')}${FOOTER}` + body: `${HEADER}${batch.join(',')}${FOOTER}`, + signal: controller.signal, }) .then((res) => { + clearTimeout(timerId) let httpErr = null; if (res.status !== 200 && res.status !== 201) { httpErr = 'HTTP log shipping failed: ' + res.statusCode; @@ -302,22 +313,9 @@ class SeqLogger { } }) .catch((err) => { - console.log('logger error', err); + clearTimeout(timerId) reject(err); }) - // TODO: not migrated yet - // req.on("socket", (socket) => { - // if (socket.listeners("timeout").length == 0) { - // socket.on("timeout", () => { - // req.destroy(); - // if (attempts > this._maxRetries) { - // return reject('HTTP log shipping failed, reached timeout (' + this._requestTimeout + ' ms)') - // } else { - // return setTimeout(() => sendRequest(batch, bytes), this._retryDelay); - // } - // }); - // } - // }); } return sendRequest(batch, bytes); From 8e220666e7f9821f2052ba5b417f741527990720 Mon Sep 17 00:00:00 2001 From: Uria Rogel Date: Thu, 27 Apr 2023 13:08:58 +0200 Subject: [PATCH 3/8] resolved removed agent reference --- seq_logger.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/seq_logger.js b/seq_logger.js index f14d82c..96e816c 100644 --- a/seq_logger.js +++ b/seq_logger.js @@ -92,9 +92,7 @@ class SeqLogger { this._closed = true; this._clearTimer(); - return this.flush().then(() => { - this._httpAgent.destroy(); - }); + return this.flush().then(() => {}); } /** From 38191aefb66d59e8861cced3436f18060d16bd28 Mon Sep 17 00:00:00 2001 From: Uria Rogel Date: Thu, 27 Apr 2023 13:12:46 +0200 Subject: [PATCH 4/8] node-fetch polyfill --- seq_logger.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/seq_logger.js b/seq_logger.js index 96e816c..d28425a 100644 --- a/seq_logger.js +++ b/seq_logger.js @@ -1,7 +1,8 @@ "use strict"; -const NodeBlob = require('buffer').Blob -const GlobalBlob = Blob !== undefined ? Blob : NodeBlob +const GlobalBlob = typeof Blob !== 'undefined' ? Blob : require('buffer').Blob; +const fetchApi = typeof fetch === 'undefined' ? require('node-fetch') : fetch; + const HEADER = '{"Events":['; const FOOTER = "]}"; const HEADER_FOOTER_BYTES = (new GlobalBlob([HEADER])).size + (new GlobalBlob([FOOTER])).size From 7145e73fed42f97d14367e308712a4ed277aad1c Mon Sep 17 00:00:00 2001 From: Uria Rogel Date: Thu, 27 Apr 2023 14:14:00 +0200 Subject: [PATCH 5/8] polyfill for AbortController Better naming for the polyfilled apis --- seq_logger.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/seq_logger.js b/seq_logger.js index d28425a..9d0e9fa 100644 --- a/seq_logger.js +++ b/seq_logger.js @@ -1,11 +1,12 @@ "use strict"; -const GlobalBlob = typeof Blob !== 'undefined' ? Blob : require('buffer').Blob; -const fetchApi = typeof fetch === 'undefined' ? require('node-fetch') : fetch; +const SafeGlobalBlob = typeof Blob !== 'undefined' ? Blob : require('buffer').Blob; +const safeGlobalFetch = typeof fetch !== 'undefined' ? require('node-fetch') : fetch; +const SafeGlobalAbortController = typeof AbortController !== 'undefined' ? require('abort-controller') : AbortController const HEADER = '{"Events":['; const FOOTER = "]}"; -const HEADER_FOOTER_BYTES = (new GlobalBlob([HEADER])).size + (new GlobalBlob([FOOTER])).size +const HEADER_FOOTER_BYTES = (new SafeGlobalBlob([HEADER])).size + (new SafeGlobalBlob([FOOTER])).size; class SeqLogger { constructor(config) { @@ -240,12 +241,12 @@ class SeqLogger { messageTemplate: "[seq] Circular structure found" }); } - var jsonLen = new GlobalBlob([json]).size; + var jsonLen = new SafeGlobalBlob([json]).size; if (jsonLen > this._eventSizeLimit) { this._onError("[seq] Event body is larger than " + this._eventSizeLimit + " bytes: " + json); this._queue[i] = next = this._eventTooLargeErrorEvent(next); json = JSON.stringify(next); - jsonLen = new GlobalBlob([json]).size; + jsonLen = new SafeGlobalBlob([json]).size; } // Always try to send a batch of at least one event, even if the batch size is @@ -274,7 +275,7 @@ class SeqLogger { return new Promise((resolve, reject) => { const sendRequest = (batch, bytes) => { - const controller = new AbortController() + const controller = new SafeGlobalAbortController(); attempts++; const timerId = setTimeout(() => { controller.abort() @@ -285,7 +286,7 @@ class SeqLogger { } }, this._requestTimeout) - fetch(this._endpoint, { + safeGlobalFetch(this._endpoint, { keepalive: true, method: "POST", headers: { From 2c009f051ff0d7ff4bbda3da19135a9fb89cc189 Mon Sep 17 00:00:00 2001 From: Uria Rogel Date: Thu, 27 Apr 2023 14:14:10 +0200 Subject: [PATCH 6/8] cosmetic --- seq_logger.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/seq_logger.js b/seq_logger.js index 9d0e9fa..f9be530 100644 --- a/seq_logger.js +++ b/seq_logger.js @@ -278,13 +278,13 @@ class SeqLogger { const controller = new SafeGlobalAbortController(); attempts++; const timerId = setTimeout(() => { - controller.abort() + controller.abort(); if (attempts > this._maxRetries) { - reject('HTTP log shipping failed, reached timeout (' + this._requestTimeout + ' ms)') + reject('HTTP log shipping failed, reached timeout (' + this._requestTimeout + ' ms)'); } else { setTimeout(() => sendRequest(batch, bytes), this._retryDelay); } - }, this._requestTimeout) + }, this._requestTimeout); safeGlobalFetch(this._endpoint, { keepalive: true, @@ -298,7 +298,7 @@ class SeqLogger { signal: controller.signal, }) .then((res) => { - clearTimeout(timerId) + clearTimeout(timerId); let httpErr = null; if (res.status !== 200 && res.status !== 201) { httpErr = 'HTTP log shipping failed: ' + res.statusCode; @@ -313,7 +313,7 @@ class SeqLogger { } }) .catch((err) => { - clearTimeout(timerId) + clearTimeout(timerId); reject(err); }) } From 183369b7fb29037bbed8feceffdc7247969fd008 Mon Sep 17 00:00:00 2001 From: Uria Rogel Date: Thu, 27 Apr 2023 14:35:56 +0200 Subject: [PATCH 7/8] same conditional format for all polyfills --- seq_logger.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seq_logger.js b/seq_logger.js index f9be530..a0979ab 100644 --- a/seq_logger.js +++ b/seq_logger.js @@ -1,8 +1,8 @@ "use strict"; const SafeGlobalBlob = typeof Blob !== 'undefined' ? Blob : require('buffer').Blob; -const safeGlobalFetch = typeof fetch !== 'undefined' ? require('node-fetch') : fetch; -const SafeGlobalAbortController = typeof AbortController !== 'undefined' ? require('abort-controller') : AbortController +const safeGlobalFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch'); +const SafeGlobalAbortController = typeof AbortController !== 'undefined' ? AbortController : require('abort-controller'); const HEADER = '{"Events":['; const FOOTER = "]}"; From ee540a3d144a6a750c32422bea57b0a095b2a802 Mon Sep 17 00:00:00 2001 From: Uria Rogel Date: Thu, 27 Apr 2023 17:16:10 +0200 Subject: [PATCH 8/8] optional polyfills with try-catch --- seq_logger.js | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/seq_logger.js b/seq_logger.js index a0979ab..ef49a6e 100644 --- a/seq_logger.js +++ b/seq_logger.js @@ -1,8 +1,24 @@ "use strict"; +var NodeBlob, NodeFetch, NodeAbortController +try { + NodeBlob = require('buffer').Blob +} catch (error) { + console.log('missing buffer') +} +try { + NodeFetch = require('node-fetch') +} catch (error) { + console.log('missing node-fetch') +} +try { + NodeAbortController = require('abort-controller') +} catch (error) { + console.log('missing abort-controller') +} -const SafeGlobalBlob = typeof Blob !== 'undefined' ? Blob : require('buffer').Blob; -const safeGlobalFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch'); -const SafeGlobalAbortController = typeof AbortController !== 'undefined' ? AbortController : require('abort-controller'); +const SafeGlobalBlob = typeof Blob !== 'undefined' ? Blob : NodeBlob; +const safeGlobalFetch = typeof fetch !== 'undefined' ? fetch : NodeFetch; +const SafeGlobalAbortController = typeof AbortController !== 'undefined' ? AbortController : NodeAbortController; const HEADER = '{"Events":['; const FOOTER = "]}";