Use Wikipedia main sites with specific CSS selectors to get data

Author: Nikolai

EventService.js

@@ -5,7 +5,7 @@
  * MIT Licensed.
  */
 
-const https = require('https');
+const { https } = require('follow-redirects');
 
 /**
  * Service object to fetch XML data from Wikipedia.
@@ -15,23 +15,23 @@ class EventService {
     /*
      * ### Instance ###
      */
-    _xml = '';
+    _html = '';
     _resolve = null;
     _reject = null;
 
     /*
      * ### Listener ###
      */
     _onData(chunk) {
-        this._xml += chunk;
+        this._html += chunk;
     }
 
     _onError(error) {
         this._reject(error);
     }
 
     _onEnd() {
-        this._resolve(this._xml);
+        this._resolve(this._html);
     }
 
     _onResponse(response) {
@@ -48,7 +48,7 @@ class EventService {
     _promiseExecutor(language) {
 
         // Wiki URL
-        const url = `https://${language}.wikipedia.org/w/api.php?action=featuredfeed&feed=onthisday`;
+        const url = `https://${language}.wikipedia.org`;
 
         // Request data
         https.get(url, response => this._onResponse(response));
@@ -57,7 +57,7 @@ class EventService {
     /*
      * ### Public API ###
      */
-    async getXml(language) {
+    async getHtml(language) {
 
         // Create and return promise
         return new Promise((resolve, reject) => {

MMM-OnThisDay.js

@@ -21,10 +21,9 @@ const moduleDefinition = {
     requiresVersion: '2.1.0', // Required version of MagicMirror
 
     /**
-     * Available languages, means that there is data to fetch for the node_helper at
-     * https://${language}.wikipedia.org/w/api.php?action=featuredfeed&feed=onthisday
+     * Available languages.
      */
-    availableLanguages: ['en', 'de'],
+    availableLanguages: ['en', 'de', 'fr'],
     usedLanguage: 'en', // Fallback
 
     /**
@@ -52,6 +51,7 @@ const moduleDefinition = {
         return {
             en: 'translation/en.json',
             de: 'translation/de.json',
+            //fr: 'TODO'
         }
     },
 
@@ -112,13 +112,9 @@ const moduleDefinition = {
                 return;
             }
 
-            // Set title
+            // Set content
             this.title = payload.title;
-
-            // Unescape
-            const domParser = new DOMParser();
-            const doc = domParser.parseFromString(payload.events, 'text/html');
-            this.events = doc.documentElement.textContent;
+            this.events = payload.events;
 
             // Update module
             Log.info('Update DOM with new title and events ...');

node_helper.js

@@ -7,8 +7,24 @@
 
 const NodeHelper = require('node_helper');
 const EventService = require('./EventService');
+const { JSDOM } = require('jsdom');
 const Log = require('./LoggerProxy');
-const xmlParser = require('fast-xml-parser');
+
+// CSS selector to find event list on language specific Wikipedia main site
+const LANGUAGE_SPECIFIC_EVENT_SELECTOR = {
+    en: {
+        title: '#mp-otd > p',
+        events: '#mp-otd > ul'
+    },
+    de: {
+        title: '#ereignisse > .hauptseite-box-title',
+        events: '#ereignisse > .hauptseite-box-content > ul:first-of-type'
+    },
+    fr: {
+        title: '.portail-droite > .accueil_2017_cadre:nth-child(2) .mw-headline',
+        events: '.portail-droite > .accueil_2017_cadre:nth-child(2) > ul'
+    },
+};
 
 module.exports = NodeHelper.create({
 
@@ -28,34 +44,38 @@ module.exports = NodeHelper.create({
     loadEvents: async function(language) {
         Log.log('Load events ...');
 
-        // Get xml
+        // Get HTML
         const eventService = new EventService();
-        const xml = await eventService.getXml(language);
+        const html = await eventService.getHtml(language);
 
         // Return parsed data
-        return this.parseEvents(xml);
+        return this.parseEvents(html, language);
     },
 
-    parseEvents: function(xml) {
-        Log.log('Parse XML data ...');
+    parseEvents: function(html, language) {
+        Log.log('Parse HTML data ...');
 
-        // Parse XML data to json
-        const json = xmlParser.parse(xml);
+        // Create dom
+        const dom = new JSDOM(html);
+        const document = dom.window.document;
+
+        // Get title
+        const titleSelector = LANGUAGE_SPECIFIC_EVENT_SELECTOR[language].title;
+        const title = document.querySelector(titleSelector);
+
+        // Get events
+        const eventsSelector = LANGUAGE_SPECIFIC_EVENT_SELECTOR[language].events;
+        const events = document.querySelector(eventsSelector);
 
         // Check data
-        if (!json || !json.rss || !json.rss.channel || !json.rss.channel.item) {
-            Log.log('Could not parse XML.')
+        if (!events) {
+            Log.log('Could not find events in HTML.')
             return {};
         }
 
-        // Get last item
-        const items = json.rss.channel.item
-        const itemsCount = items.length;
-        const item = items[itemsCount - 1];
-
         return {
-            title: item.title,
-            events: item.description,
+            title: title ? title.innerHTML : null,
+            events: events.outerHTML,
         };
     },
 });