From e27fd2ea4c0f453eeea1a31a0c6d4491568d914b Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Tue, 25 Oct 2022 18:06:46 +0200 Subject: [PATCH 1/3] Moved server logic to separate file. A separate file makes it easier to test. Added unit tests to the cors-method. --- js/server.js | 50 +-------- js/server_functions.js | 76 +++++++++++++ tests/unit/functions/server_functions_spec.js | 103 ++++++++++++++++++ 3 files changed, 184 insertions(+), 45 deletions(-) create mode 100644 js/server_functions.js create mode 100644 tests/unit/functions/server_functions_spec.js diff --git a/js/server.js b/js/server.js index bd86219f9c..364cce4ce2 100644 --- a/js/server.js +++ b/js/server.js @@ -9,10 +9,10 @@ const path = require("path"); const ipfilter = require("express-ipfilter").IpFilter; const fs = require("fs"); const helmet = require("helmet"); -const fetch = require("fetch"); const Log = require("logger"); const Utils = require("./utils.js"); +const { cors, getConfig, getHtml, getVersion } = require("./server_functions.js"); /** * Server @@ -78,53 +78,13 @@ function Server(config) { app.use(directory, express.static(path.resolve(global.root_path + directory))); } - app.get("/cors", async function (req, res) { - // example: http://localhost:8080/cors?url=https://google.de - - try { - const reg = "^/cors.+url=(.*)"; - let url = ""; - - let match = new RegExp(reg, "g").exec(req.url); - if (!match) { - url = "invalid url: " + req.url; - Log.error(url); - res.send(url); - } else { - url = match[1]; - Log.log("cors url: " + url); - const response = await fetch(url, { headers: { "User-Agent": "Mozilla/5.0 MagicMirror/" + global.version } }); - const header = response.headers.get("Content-Type"); - const data = await response.text(); - if (header) res.set("Content-Type", header); - res.send(data); - } - } catch (error) { - Log.error(error); - res.send(error); - } - }); + app.get("/cors", async (req, res) => await cors(req, res)); - app.get("/version", function (req, res) { - res.send(global.version); - }); + app.get("/version", (req, res) => getVersion(req, res)); - app.get("/config", function (req, res) { - res.send(config); - }); + app.get("/config", (req, res) => getConfig(req, res)); - app.get("/", function (req, res) { - let html = fs.readFileSync(path.resolve(`${global.root_path}/index.html`), { encoding: "utf8" }); - html = html.replace("#VERSION#", global.version); - - let configFile = "config/config.js"; - if (typeof global.configuration_file !== "undefined") { - configFile = global.configuration_file; - } - html = html.replace("#CONFIG_FILE#", configFile); - - res.send(html); - }); + app.get("/", (req, res) => getHtml(req, res)); return { app, diff --git a/js/server_functions.js b/js/server_functions.js new file mode 100644 index 0000000000..e9b7961787 --- /dev/null +++ b/js/server_functions.js @@ -0,0 +1,76 @@ +const fetch = require("./fetch"); +const fs = require("fs"); +const path = require("path"); +const Log = require("logger"); + +/** + * Gets the config. + * + * @param {Request} req - the request + * @param {Response} res - the result + */ +function getConfig(req, res) { + res.send(config); +} + +/** + * A method that forewards HTTP Get-methods to the internet to avoid CORS-errors. + * + * @param {Request} req - the request + * @param {Response} res - the result + */ +async function cors(req, res) { + try { + const reg = "^/cors.+url=(.*)"; + let url = ""; + + let match = new RegExp(reg, "g").exec(req.url); + if (!match) { + url = "invalid url: " + req.url; + Log.error(url); + res.send(url); + } else { + url = match[1]; + Log.log("cors url: " + url); + const response = await fetch(url, { headers: { "User-Agent": "Mozilla/5.0 MagicMirror/" + global.version } }); + const header = response.headers.get("Content-Type"); + const data = await response.text(); + if (header) res.set("Content-Type", header); + res.send(data); + } + } catch (error) { + Log.error(error); + res.send(error); + } +} + +/** + * Gets the HTML to display the magic mirror. + * + * @param {Request} req - the request + * @param {Response} res - the result + */ +function getHtml(req, res) { + let html = fs.readFileSync(path.resolve(`${global.root_path}/index.html`), { encoding: "utf8" }); + html = html.replace("#VERSION#", global.version); + + let configFile = "config/config.js"; + if (typeof global.configuration_file !== "undefined") { + configFile = global.configuration_file; + } + html = html.replace("#CONFIG_FILE#", configFile); + + res.send(html); +} + +/** + * Gets the MacigMirror version. + * + * @param {Request} req - the request + * @param {Response} res - the result + */ +function getVersion(req, res) { + res.send(global.version); +} + +module.exports = { cors, getConfig, getHtml, getVersion }; diff --git a/tests/unit/functions/server_functions_spec.js b/tests/unit/functions/server_functions_spec.js new file mode 100644 index 0000000000..eb8944d2ab --- /dev/null +++ b/tests/unit/functions/server_functions_spec.js @@ -0,0 +1,103 @@ +const { cors } = require("../../../js/server_functions"); + +describe("server_functions tests", () => { + describe("The cors method", () => { + let fetchResponse; + let fetchResponseHeadersGet; + let fetchResponseHeadersText; + let corsResponse; + let request; + + beforeEach(() => { + fetchResponse = new Response(); + global.fetch = jest.fn(() => Promise.resolve(fetchResponse)); + fetchResponseHeadersGet = jest.spyOn(fetchResponse.headers, "get"); + fetchResponseHeadersText = jest.spyOn(fetchResponse, "text"); + + corsResponse = { + set: jest.fn(() => {}), + send: jest.fn(() => {}) + }; + + request = { + url: `/cors?url=www.test.com` + }; + }); + + test("Calls correct URL once", async () => { + const urlToCall = "ttp://www.test.com/path?param1=value1"; + request.url = `/cors?url=${urlToCall}`; + + await cors(request, corsResponse); + + expect(global.fetch.mock.calls.length).toBe(1); + expect(global.fetch.mock.calls[0][0]).toBe(urlToCall); + }); + + test("Forewards Content-Type if json", async () => { + fetchResponseHeadersGet.mockImplementation(() => "json"); + + await cors(request, corsResponse); + + expect(fetchResponseHeadersGet.mock.calls.length).toBe(1); + expect(fetchResponseHeadersGet.mock.calls[0][0]).toBe("Content-Type"); + + expect(corsResponse.set.mock.calls.length).toBe(1); + expect(corsResponse.set.mock.calls[0][0]).toBe("Content-Type"); + expect(corsResponse.set.mock.calls[0][1]).toBe("json"); + }); + + test("Forewards Content-Type if xml", async () => { + fetchResponseHeadersGet.mockImplementation(() => "xml"); + + await cors(request, corsResponse); + + expect(fetchResponseHeadersGet.mock.calls.length).toBe(1); + expect(fetchResponseHeadersGet.mock.calls[0][0]).toBe("Content-Type"); + + expect(corsResponse.set.mock.calls.length).toBe(1); + expect(corsResponse.set.mock.calls[0][0]).toBe("Content-Type"); + expect(corsResponse.set.mock.calls[0][1]).toBe("xml"); + }); + + test("Sends correct data from response", async () => { + const responseData = "some data"; + fetchResponseHeadersText.mockImplementation(() => responseData); + + let sentData; + corsResponse.send = jest.fn((input) => { + sentData = input; + }); + + await cors(request, corsResponse); + + expect(fetchResponseHeadersText.mock.calls.length).toBe(1); + expect(sentData).toBe(responseData); + }); + + test("Sends error data from response", async () => { + const error = new Error("error data"); + fetchResponseHeadersText.mockImplementation(() => { + throw error; + }); + + let sentData; + corsResponse.send = jest.fn((input) => { + sentData = input; + }); + + await cors(request, corsResponse); + + expect(fetchResponseHeadersText.mock.calls.length).toBe(1); + expect(sentData).toBe(error); + }); + + test("Sends user agent by default", async () => { + await cors(request, corsResponse); + + expect(global.fetch.mock.calls.length).toBe(1); + expect(global.fetch.mock.calls[0][1]).toHaveProperty("headers"); + expect(global.fetch.mock.calls[0][1].headers).toHaveProperty("User-Agent"); + }); + }); +}); From 1c8ea72e1e4e1cd6873a74eaae78a43c237d84cc Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Tue, 25 Oct 2022 19:40:44 +0200 Subject: [PATCH 2/3] Added functionality for sending and recieving HTTP-headers. This is required for some weather-providers, and will probably be useful for other services. --- CHANGELOG.md | 3 +- js/server_functions.js | 61 ++++++++++++-- tests/unit/functions/server_functions_spec.js | 81 ++++++++++++++++--- 3 files changed, 128 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5fbff27a2..d68df1a006 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). _This release is scheduled to be released on 2023-01-01._ -Special thanks to: @rejas, @sdetweil +Special thanks to: @rejas, @sdetweil, @MagMar94 ### Added @@ -29,6 +29,7 @@ Special thanks to: @rejas, @sdetweil - Rework weather module - Use fetch instead of XMLHttpRequest in weatherprovider - Use unix() method for parsing times, fix suntimes on the way +- The `cors`-method in `server.js` now supports sending and recieving HTTP headers. ### Fixed diff --git a/js/server_functions.js b/js/server_functions.js index e9b7961787..e7e9a0d024 100644 --- a/js/server_functions.js +++ b/js/server_functions.js @@ -16,26 +16,37 @@ function getConfig(req, res) { /** * A method that forewards HTTP Get-methods to the internet to avoid CORS-errors. * + * Example input request url: /cors?sendheaders=header1:value1,header2:value2&expectedheaders=header1,header2&url=http://www.test.com/path?param1=value1 + * + * Only the url-param of the input request url is required. + * * @param {Request} req - the request * @param {Response} res - the result */ async function cors(req, res) { try { - const reg = "^/cors.+url=(.*)"; + const urlRegEx = "url=(.+?)$"; let url = ""; - let match = new RegExp(reg, "g").exec(req.url); + const match = new RegExp(urlRegEx, "g").exec(req.url); if (!match) { url = "invalid url: " + req.url; Log.error(url); res.send(url); } else { url = match[1]; + + const headersToSend = getHeadersToSend(req.url); + const expectedRecievedHeaders = geExpectedRecievedHeaders(req.url); + Log.log("cors url: " + url); - const response = await fetch(url, { headers: { "User-Agent": "Mozilla/5.0 MagicMirror/" + global.version } }); - const header = response.headers.get("Content-Type"); + const response = await fetch(url, { headers: headersToSend }); + + for (const header of expectedRecievedHeaders) { + const headerValue = response.headers.get(header); + if (header) res.set(header, headerValue); + } const data = await response.text(); - if (header) res.set("Content-Type", header); res.send(data); } } catch (error) { @@ -44,6 +55,46 @@ async function cors(req, res) { } } +/** + * Gets headers and values to attatch to the web request. + * + * @param {string} url - The url containing the headers and values to send. + * @returns {object} An object specifying name and value of the headers. + */ +function getHeadersToSend(url) { + const headersToSend = { "User-Agent": "Mozilla/5.0 MagicMirror/" + global.version }; + const headersToSendMatch = new RegExp("sendheaders=(.+?)(&|$)", "g").exec(url); + if (headersToSendMatch) { + const headers = headersToSendMatch[1].split(","); + for (const header of headers) { + const keyValue = header.split(":"); + if (keyValue.length !== 2) { + throw new Error(`$Invalid format for header ${header}`); + } + headersToSend[keyValue[0]] = decodeURIComponent(keyValue[1]); + } + } + return headersToSend; +} + +/** + * Gets the headers expected from the response. + * + * @param {string} url - The url containing the expected headers from the response. + * @returns {string[]} headers - The name of the expected headers. + */ +function geExpectedRecievedHeaders(url) { + const expectedRecievedHeaders = ["Content-Type"]; + const expectedRecievedHeadersMatch = new RegExp("expectedheaders=(.+?)(&|$)", "g").exec(url); + if (expectedRecievedHeadersMatch) { + const headers = expectedRecievedHeadersMatch[1].split(","); + for (const header of headers) { + expectedRecievedHeaders.push(header); + } + } + return expectedRecievedHeaders; +} + /** * Gets the HTML to display the magic mirror. * diff --git a/tests/unit/functions/server_functions_spec.js b/tests/unit/functions/server_functions_spec.js index eb8944d2ab..e90134be4a 100644 --- a/tests/unit/functions/server_functions_spec.js +++ b/tests/unit/functions/server_functions_spec.js @@ -1,4 +1,5 @@ const { cors } = require("../../../js/server_functions"); +const nodeVersion = process.version.match(/^v(\d+)\.*/)[1]; describe("server_functions tests", () => { describe("The cors method", () => { @@ -8,11 +9,37 @@ describe("server_functions tests", () => { let corsResponse; let request; + let nodefetch; + if (nodeVersion < 18) { + jest.mock("node-fetch"); + nodefetch = require("node-fetch"); + } + let fetchMock; + beforeEach(() => { - fetchResponse = new Response(); - global.fetch = jest.fn(() => Promise.resolve(fetchResponse)); - fetchResponseHeadersGet = jest.spyOn(fetchResponse.headers, "get"); - fetchResponseHeadersText = jest.spyOn(fetchResponse, "text"); + if (nodeVersion >= 18) { + fetchResponse = new Response(); + global.fetch = jest.fn(() => Promise.resolve(fetchResponse)); + fetchResponseHeadersGet = jest.spyOn(fetchResponse.headers, "get"); + fetchResponseHeadersText = jest.spyOn(fetchResponse, "text"); + + fetchMock = global.fetch; + } else { + nodefetch.mockReset(); + + fetchResponseHeadersGet = jest.fn(() => {}); + fetchResponseHeadersText = jest.fn(() => {}); + fetchResponse = { + headers: { + get: fetchResponseHeadersGet + }, + text: fetchResponseHeadersText + }; + jest.mock("node-fetch", () => jest.fn()); + nodefetch.mockImplementation(() => fetchResponse); + + fetchMock = nodefetch; + } corsResponse = { set: jest.fn(() => {}), @@ -25,13 +52,13 @@ describe("server_functions tests", () => { }); test("Calls correct URL once", async () => { - const urlToCall = "ttp://www.test.com/path?param1=value1"; + const urlToCall = "http://www.test.com/path?param1=value1"; request.url = `/cors?url=${urlToCall}`; await cors(request, corsResponse); - expect(global.fetch.mock.calls.length).toBe(1); - expect(global.fetch.mock.calls[0][0]).toBe(urlToCall); + expect(fetchMock.mock.calls.length).toBe(1); + expect(fetchMock.mock.calls[0][0]).toBe(urlToCall); }); test("Forewards Content-Type if json", async () => { @@ -92,12 +119,44 @@ describe("server_functions tests", () => { expect(sentData).toBe(error); }); - test("Sends user agent by default", async () => { + test("Fetches with user agent by default", async () => { await cors(request, corsResponse); - expect(global.fetch.mock.calls.length).toBe(1); - expect(global.fetch.mock.calls[0][1]).toHaveProperty("headers"); - expect(global.fetch.mock.calls[0][1].headers).toHaveProperty("User-Agent"); + expect(fetchMock.mock.calls.length).toBe(1); + expect(fetchMock.mock.calls[0][1]).toHaveProperty("headers"); + expect(fetchMock.mock.calls[0][1].headers).toHaveProperty("User-Agent"); + }); + + test("Fetches with specified headers", async () => { + const headersParam = "sendheaders=header1:value1,header2:value2"; + const urlParam = "http://www.test.com/path?param1=value1"; + request.url = `/cors?${headersParam}&url=${urlParam}`; + + await cors(request, corsResponse); + + expect(fetchMock.mock.calls.length).toBe(1); + expect(fetchMock.mock.calls[0][1]).toHaveProperty("headers"); + expect(fetchMock.mock.calls[0][1].headers).toHaveProperty("header1", "value1"); + expect(fetchMock.mock.calls[0][1].headers).toHaveProperty("header2", "value2"); + }); + + test("Sends specified headers", async () => { + fetchResponseHeadersGet.mockImplementation((input) => input.replace("header", "value")); + + const expectedheaders = "expectedheaders=header1,header2"; + const urlParam = "http://www.test.com/path?param1=value1"; + request.url = `/cors?${expectedheaders}&url=${urlParam}`; + + await cors(request, corsResponse); + + expect(fetchMock.mock.calls.length).toBe(1); + expect(fetchMock.mock.calls[0][1]).toHaveProperty("headers"); + expect(corsResponse.set.mock.calls.length).toBe(3); + expect(corsResponse.set.mock.calls[0][0]).toBe("Content-Type"); + expect(corsResponse.set.mock.calls[1][0]).toBe("header1"); + expect(corsResponse.set.mock.calls[1][1]).toBe("value1"); + expect(corsResponse.set.mock.calls[2][0]).toBe("header2"); + expect(corsResponse.set.mock.calls[2][1]).toBe("value2"); }); }); }); From 68e321b04daa62ec247916dd0d73415603fc6bec Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Sun, 30 Oct 2022 16:30:56 +0100 Subject: [PATCH 3/3] Small changes after merging with develop-branch. --- js/server_functions.js | 2 +- tests/unit/functions/server_functions_spec.js | 45 +++++++------------ 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/js/server_functions.js b/js/server_functions.js index b74393e2c7..f210a8b848 100644 --- a/js/server_functions.js +++ b/js/server_functions.js @@ -18,7 +18,7 @@ function getConfig(req, res) { * * Example input request url: /cors?sendheaders=header1:value1,header2:value2&expectedheaders=header1,header2&url=http://www.test.com/path?param1=value1 * - * Only the url-param of the input request url is required. + * Only the url-param of the input request url is required. It must be the last parameter. * * @param {Request} req - the request * @param {Response} res - the result diff --git a/tests/unit/functions/server_functions_spec.js b/tests/unit/functions/server_functions_spec.js index e90134be4a..3548e38a0e 100644 --- a/tests/unit/functions/server_functions_spec.js +++ b/tests/unit/functions/server_functions_spec.js @@ -1,5 +1,4 @@ const { cors } = require("../../../js/server_functions"); -const nodeVersion = process.version.match(/^v(\d+)\.*/)[1]; describe("server_functions tests", () => { describe("The cors method", () => { @@ -9,37 +8,25 @@ describe("server_functions tests", () => { let corsResponse; let request; - let nodefetch; - if (nodeVersion < 18) { - jest.mock("node-fetch"); - nodefetch = require("node-fetch"); - } + jest.mock("node-fetch"); + let nodefetch = require("node-fetch"); let fetchMock; beforeEach(() => { - if (nodeVersion >= 18) { - fetchResponse = new Response(); - global.fetch = jest.fn(() => Promise.resolve(fetchResponse)); - fetchResponseHeadersGet = jest.spyOn(fetchResponse.headers, "get"); - fetchResponseHeadersText = jest.spyOn(fetchResponse, "text"); - - fetchMock = global.fetch; - } else { - nodefetch.mockReset(); - - fetchResponseHeadersGet = jest.fn(() => {}); - fetchResponseHeadersText = jest.fn(() => {}); - fetchResponse = { - headers: { - get: fetchResponseHeadersGet - }, - text: fetchResponseHeadersText - }; - jest.mock("node-fetch", () => jest.fn()); - nodefetch.mockImplementation(() => fetchResponse); - - fetchMock = nodefetch; - } + nodefetch.mockReset(); + + fetchResponseHeadersGet = jest.fn(() => {}); + fetchResponseHeadersText = jest.fn(() => {}); + fetchResponse = { + headers: { + get: fetchResponseHeadersGet + }, + text: fetchResponseHeadersText + }; + jest.mock("node-fetch", () => jest.fn()); + nodefetch.mockImplementation(() => fetchResponse); + + fetchMock = nodefetch; corsResponse = { set: jest.fn(() => {}),