From 8c37d581549d5396c5a6efb475fee09ab812d8a2 Mon Sep 17 00:00:00 2001 From: Kristjan ESPERANTO <35647502+KristjanESPERANTO@users.noreply.github.com> Date: Fri, 14 Nov 2025 00:00:29 +0100 Subject: [PATCH] [calendar] refactor: migrate CalendarFetcher to ES6 class and improve error handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Convert CalendarFetcher from legacy constructor function pattern to ES6 class, which simplifies future migration to ES modules. 2. Implement targeted HTTP error handling with smart retry strategies for common calendar feed issues: - 401/403: Extended retry delay (5× interval, min 30 min) - 429: Retry-After header parsing with 15 min fallback - 5xx: Exponential backoff (2^count, max 3 retries) - 4xx: Extended retry (2× interval, min 15 min) - Add serverErrorCount tracking for exponential backoff - Error messages now include specific HTTP status codes and calculated retry delays for better debugging and user feedback Previously, CalendarFetcher did not respond appropriately to HTTP errors, continuing to hammer endpoints without backoff, potentially overloading servers and triggering rate limits. This refactoring implements respectful retry strategies that adapt to server responses and reduce unnecessary load. --- CHANGELOG.md | 1 + js/node_helper.js | 7 +- modules/default/calendar/calendarfetcher.js | 260 +++++++++++++------- modules/default/calendar/debug.js | 2 +- modules/default/calendar/node_helper.js | 10 +- 5 files changed, 180 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58f13a330c..9d96561af8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ planned for 2026-01-01 - [core] configure cspell to check default modules only and fix typos (#3955) - [core] refactor: replace `XMLHttpRequest` with `fetch` in `translator.js` (#3950) - [tests] migrate e2e tests to Playwright (#3950) +- [calendar] refactor: migrate CalendarFetcher to ES6 class and improve error handling (#3958) ### Fixed diff --git a/js/node_helper.js b/js/node_helper.js index bc91dd52a4..cbe30edda0 100644 --- a/js/node_helper.js +++ b/js/node_helper.js @@ -113,8 +113,11 @@ NodeHelper.checkFetchError = function (error) { let error_type = "MODULE_ERROR_UNSPECIFIED"; if (error.code === "EAI_AGAIN") { error_type = "MODULE_ERROR_NO_CONNECTION"; - } else if (error.message === "Unauthorized") { - error_type = "MODULE_ERROR_UNAUTHORIZED"; + } else { + const message = typeof error.message === "string" ? error.message.toLowerCase() : ""; + if (message.includes("unauthorized") || message.includes("http 401") || message.includes("http 403")) { + error_type = "MODULE_ERROR_UNAUTHORIZED"; + } } return error_type; }; diff --git a/modules/default/calendar/calendarfetcher.js b/modules/default/calendar/calendarfetcher.js index 60a0ec4d0a..5f5d5bfa74 100644 --- a/modules/default/calendar/calendarfetcher.js +++ b/modules/default/calendar/calendarfetcher.js @@ -1,131 +1,207 @@ const https = require("node:https"); const ical = require("node-ical"); const Log = require("logger"); -const NodeHelper = require("node_helper"); const CalendarFetcherUtils = require("./calendarfetcherutils"); const { getUserAgent } = require("#server_functions"); -const { scheduleTimer } = require("#module_functions"); + +const FIFTEEN_MINUTES = 15 * 60 * 1000; +const THIRTY_MINUTES = 30 * 60 * 1000; +const MAX_SERVER_BACKOFF = 3; /** - * - * @param {string} url The url of the calendar to fetch - * @param {number} reloadInterval Time in ms the calendar is fetched again - * @param {string[]} excludedEvents An array of words / phrases from event titles that will be excluded from being shown. - * @param {number} maximumEntries The maximum number of events fetched. - * @param {number} maximumNumberOfDays The maximum number of days an event should be in the future. - * @param {object} auth The object containing options for authentication against the calendar. - * @param {boolean} includePastEvents If true events from the past maximumNumberOfDays will be fetched too - * @param {boolean} selfSignedCert If true, the server certificate is not verified against the list of supplied CAs. + * CalendarFetcher - Fetches and parses iCal calendar data with MagicMirror-focused error handling * @class */ -const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEntries, maximumNumberOfDays, auth, includePastEvents, selfSignedCert) { - let reloadTimer = null; - let events = []; +class CalendarFetcher { - let fetchFailedCallback = function () {}; - let eventsReceivedCallback = function () {}; + /** + * Creates a new CalendarFetcher instance + * @param {string} url - The URL of the calendar to fetch + * @param {number} reloadInterval - Time in ms between fetches + * @param {string[]} excludedEvents - Event titles to exclude + * @param {number} maximumEntries - Maximum number of events to return + * @param {number} maximumNumberOfDays - Maximum days in the future to fetch + * @param {object} auth - Authentication options {method: 'basic'|'bearer', user, pass} + * @param {boolean} includePastEvents - Whether to include past events + * @param {boolean} selfSignedCert - Whether to accept self-signed certificates + */ + constructor (url, reloadInterval, excludedEvents, maximumEntries, maximumNumberOfDays, auth, includePastEvents, selfSignedCert) { + this.url = url; + this.reloadInterval = reloadInterval; + this.excludedEvents = excludedEvents; + this.maximumEntries = maximumEntries; + this.maximumNumberOfDays = maximumNumberOfDays; + this.auth = auth; + this.includePastEvents = includePastEvents; + this.selfSignedCert = selfSignedCert; + + this.events = []; + this.reloadTimer = null; + this.serverErrorCount = 0; + this.fetchFailedCallback = () => {}; + this.eventsReceivedCallback = () => {}; + } /** - * Initiates calendar fetch. + * Clears any pending reload timer */ - const fetchCalendar = () => { - clearTimeout(reloadTimer); - reloadTimer = null; - let httpsAgent = null; - let headers = { - "User-Agent": getUserAgent() - }; + clearReloadTimer () { + if (this.reloadTimer) { + clearTimeout(this.reloadTimer); + this.reloadTimer = null; + } + } - if (selfSignedCert) { - httpsAgent = new https.Agent({ - rejectUnauthorized: false - }); + /** + * Schedules the next fetch respecting MagicMirror test mode + * @param {number} delay - Delay in milliseconds + */ + scheduleNextFetch (delay) { + const nextDelay = Math.max(delay || this.reloadInterval, this.reloadInterval); + if (process.env.mmTestMode === "true") { + return; } - if (auth) { - if (auth.method === "bearer") { - headers.Authorization = `Bearer ${auth.pass}`; + this.reloadTimer = setTimeout(() => this.fetchCalendar(), nextDelay); + } + + /** + * Builds the options object for fetch + * @returns {object} Options object containing headers (and agent if needed) + */ + getRequestOptions () { + const headers = { "User-Agent": getUserAgent() }; + const options = { headers }; + + if (this.selfSignedCert) { + options.agent = new https.Agent({ rejectUnauthorized: false }); + } + + if (this.auth) { + if (this.auth.method === "bearer") { + headers.Authorization = `Bearer ${this.auth.pass}`; } else { - headers.Authorization = `Basic ${Buffer.from(`${auth.user}:${auth.pass}`).toString("base64")}`; + headers.Authorization = `Basic ${Buffer.from(`${this.auth.user}:${this.auth.pass}`).toString("base64")}`; } } - fetch(url, { headers: headers, agent: httpsAgent }) - .then(NodeHelper.checkFetchStatus) - .then((response) => response.text()) - .then((responseData) => { - let data = []; - - try { - data = ical.parseICS(responseData); - Log.debug(`parsed data=${JSON.stringify(data, null, 2)}`); - events = CalendarFetcherUtils.filterEvents(data, { - excludedEvents, - includePastEvents, - maximumEntries, - maximumNumberOfDays - }); - } catch (error) { - fetchFailedCallback(this, error); - scheduleTimer(reloadTimer, reloadInterval, fetchCalendar); - return; - } - this.broadcastEvents(); - scheduleTimer(reloadTimer, reloadInterval, fetchCalendar); - }) - .catch((error) => { - fetchFailedCallback(this, error); - scheduleTimer(reloadTimer, reloadInterval, fetchCalendar); - }); - }; - - /* public methods */ + return options; + } /** - * Initiate fetchCalendar(); + * Parses the Retry-After header value + * @param {string} retryAfter - The Retry-After header value + * @returns {number|null} Milliseconds to wait or null if parsing failed */ - this.startFetch = function () { - fetchCalendar(); - }; + parseRetryAfter (retryAfter) { + const seconds = Number(retryAfter); + if (!Number.isNaN(seconds) && seconds >= 0) { + return seconds * 1000; + } + + const retryDate = Date.parse(retryAfter); + if (!Number.isNaN(retryDate)) { + return Math.max(0, retryDate - Date.now()); + } + + return null; + } /** - * Broadcast the existing events. + * Determines the retry delay for a non-ok response + * @param {Response} response - The fetch Response object + * @returns {{delay: number, error: Error}} Error describing the issue and computed retry delay */ - this.broadcastEvents = function () { - Log.info(`Fetcher: Broadcasting ${events.length} events from ${url}.`); - eventsReceivedCallback(this); - }; + getDelayForResponse (response) { + const { status, statusText = "" } = response; + let delay = this.reloadInterval; + + if (status === 401 || status === 403) { + delay = Math.max(this.reloadInterval * 5, THIRTY_MINUTES); + Log.error(`${this.url} - Authentication failed (${status}). Waiting ${Math.round(delay / 60000)} minutes before retry.`); + } else if (status === 429) { + const retryAfter = response.headers.get("retry-after"); + const parsed = retryAfter ? this.parseRetryAfter(retryAfter) : null; + delay = parsed !== null ? Math.max(parsed, this.reloadInterval) : Math.max(this.reloadInterval * 2, FIFTEEN_MINUTES); + Log.warn(`${this.url} - Rate limited (429). Retrying in ${Math.round(delay / 60000)} minutes.`); + } else if (status >= 500) { + this.serverErrorCount = Math.min(this.serverErrorCount + 1, MAX_SERVER_BACKOFF); + delay = this.reloadInterval * Math.pow(2, this.serverErrorCount); + Log.error(`${this.url} - Server error (${status}). Retry #${this.serverErrorCount} in ${Math.round(delay / 60000)} minutes.`); + } else if (status >= 400) { + delay = Math.max(this.reloadInterval * 2, FIFTEEN_MINUTES); + Log.error(`${this.url} - Client error (${status}). Retrying in ${Math.round(delay / 60000)} minutes.`); + } else { + Log.error(`${this.url} - Unexpected HTTP status ${status}.`); + } + + return { + delay, + error: new Error(`HTTP ${status} ${statusText}`.trim()) + }; + } /** - * Sets the on success callback - * @param {eventsReceivedCallback} callback The on success callback. + * Fetches and processes calendar data */ - this.onReceive = function (callback) { - eventsReceivedCallback = callback; - }; + async fetchCalendar () { + this.clearReloadTimer(); + + let nextDelay = this.reloadInterval; + try { + const response = await fetch(this.url, this.getRequestOptions()); + if (!response.ok) { + const { delay, error } = this.getDelayForResponse(response); + nextDelay = delay; + this.fetchFailedCallback(this, error); + } else { + this.serverErrorCount = 0; + const responseData = await response.text(); + try { + const parsed = ical.parseICS(responseData); + Log.debug(`Parsed iCal data from ${this.url} with ${Object.keys(parsed).length} entries.`); + this.events = CalendarFetcherUtils.filterEvents(parsed, { + excludedEvents: this.excludedEvents, + includePastEvents: this.includePastEvents, + maximumEntries: this.maximumEntries, + maximumNumberOfDays: this.maximumNumberOfDays + }); + this.broadcastEvents(); + } catch (error) { + Log.error(`${this.url} - iCal parsing failed: ${error.message}`); + this.fetchFailedCallback(this, error); + } + } + } catch (error) { + Log.error(`${this.url} - Fetch failed: ${error.message}`); + this.fetchFailedCallback(this, error); + } + + this.scheduleNextFetch(nextDelay); + } /** - * Sets the on error callback - * @param {fetchFailedCallback} callback The on error callback. + * Broadcasts the current events to listeners */ - this.onError = function (callback) { - fetchFailedCallback = callback; - }; + broadcastEvents () { + Log.info(`Broadcasting ${this.events.length} events from ${this.url}.`); + this.eventsReceivedCallback(this); + } /** - * Returns the url of this fetcher. - * @returns {string} The url of this fetcher. + * Sets the callback for successful event fetches + * @param {(fetcher: CalendarFetcher) => void} callback - Called when events are received */ - this.url = function () { - return url; - }; + onReceive (callback) { + this.eventsReceivedCallback = callback; + } /** - * Returns current available events for this fetcher. - * @returns {object[]} The current available events for this fetcher. + * Sets the callback for fetch failures + * @param {(fetcher: CalendarFetcher, error: Error) => void} callback - Called when a fetch fails */ - this.events = function () { - return events; - }; -}; + onError (callback) { + this.fetchFailedCallback = callback; + } +} module.exports = CalendarFetcher; diff --git a/modules/default/calendar/debug.js b/modules/default/calendar/debug.js index e4280015c6..53a0d78046 100644 --- a/modules/default/calendar/debug.js +++ b/modules/default/calendar/debug.js @@ -26,7 +26,7 @@ Log.log("Create fetcher ..."); const fetcher = new CalendarFetcher(url, fetchInterval, [], maximumEntries, maximumNumberOfDays, auth); fetcher.onReceive(function (fetcher) { - Log.log(fetcher.events()); + Log.log(fetcher.events); process.exit(0); }); diff --git a/modules/default/calendar/node_helper.js b/modules/default/calendar/node_helper.js index 6cebb4844e..f3519aa57a 100644 --- a/modules/default/calendar/node_helper.js +++ b/modules/default/calendar/node_helper.js @@ -20,7 +20,7 @@ module.exports = NodeHelper.create({ this.sendSocketNotification("CALENDAR_ERROR", { error_type: "MODULE_ERROR_UNSPECIFIED" }); return; } - this.fetchers[key].startFetch(); + this.fetchers[key].fetchCalendar(); } }, @@ -61,7 +61,7 @@ module.exports = NodeHelper.create({ }); fetcher.onError((fetcher, error) => { - Log.error("Calendar Error. Could not fetch calendar: ", fetcher.url(), error); + Log.error("Calendar Error. Could not fetch calendar: ", fetcher.url, error); let error_type = NodeHelper.checkFetchError(error); this.sendSocketNotification("CALENDAR_ERROR", { id: identifier, @@ -76,7 +76,7 @@ module.exports = NodeHelper.create({ fetcher.broadcastEvents(); } - fetcher.startFetch(); + fetcher.fetchCalendar(); }, /** @@ -87,8 +87,8 @@ module.exports = NodeHelper.create({ broadcastEvents (fetcher, identifier) { this.sendSocketNotification("CALENDAR_EVENTS", { id: identifier, - url: fetcher.url(), - events: fetcher.events() + url: fetcher.url, + events: fetcher.events }); } });