From 90bcc1dc4db0c447bca4fa27cda62235ab14b3dc Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Fri, 28 Oct 2022 12:55:00 +0200 Subject: [PATCH 1/9] Implemented Yr as a weather provider. The Yr documentation was quite strict on using a cache together with the expires- and if-modified since-headers. As a result the weather provider has to send and recieve HTTP-headers. We use the default User-Agent for now, even though it is on a slightly different format than the one documented by Yr. --- CHANGELOG.md | 2 + modules/default/weather/providers/yr.js | 640 +++++++++++++++++++++ modules/default/weather/weatherprovider.js | 118 +++- 3 files changed, 749 insertions(+), 11 deletions(-) create mode 100644 modules/default/weather/providers/yr.js diff --git a/CHANGELOG.md b/CHANGELOG.md index e5fbff27a2..1a709ec97b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Special thanks to: @rejas, @sdetweil - Added hourlyWeather functionality to Weather.gov weather provider - Removed weatherEndpoint definition from weathergov.js (not used) - Added css class names "today" and "tomorrow" for default calendar +- Added Yr as a weather provider ### Removed @@ -29,6 +30,7 @@ Special thanks to: @rejas, @sdetweil - Rework weather module - Use fetch instead of XMLHttpRequest in weatherprovider - Use unix() method for parsing times, fix suntimes on the way + - Support HTTP headers ### Fixed diff --git a/modules/default/weather/providers/yr.js b/modules/default/weather/providers/yr.js new file mode 100644 index 0000000000..ffc8685f85 --- /dev/null +++ b/modules/default/weather/providers/yr.js @@ -0,0 +1,640 @@ +/* global WeatherProvider, WeatherObject */ + +/* MagicMirror² + * Module: Weather + * Provider: Yr.no + * + * By Magnus Marthinsen + * MIT Licensed + * + * This class is a provider for Yr.no, a norwegian sweather service. + * + * Terms of service: https://developer.yr.no/doc/TermsOfService/ + */ +WeatherProvider.register("yr", { + providerName: "Yr", + + // Set the default config properties that is specific to this provider + defaults: { + useCorsProxy: true, + apiBase: "https://api.met.no/weatherapi", + altitude: 0, + currentForecastHours: 1 //1, 6 or 12 + }, + + //Backup cache if local storage does not work + cache: { + weatherData: undefined, + stellarData: { + today: undefined, + tomorrow: undefined + } + }, + + fetchCurrentWeather() { + this.getCurrentWeather() + .then((currentWeather) => { + this.setCurrentWeather(currentWeather); + this.updateAvailable(); + }) + .catch((error) => { + Log.error(error); + throw new Error(error); + }); + }, + + async getCurrentWeather() { + const getRequests = [this.getWeatherData(), this.getStellarData()]; + const [weatherData, stellarData] = await Promise.all(getRequests); + if (!stellarData) { + Log.warn("No stelar data available."); + } + if (!weatherData.properties.timeseries || !weatherData.properties.timeseries[0]) { + Log.error("No weather data available."); + return; + } + const currentTime = moment(); + let forecast = weatherData.properties.timeseries[0]; + let closestTimeInPast = currentTime.diff(moment(forecast.time)); + for (const forecastTime of weatherData.properties.timeseries) { + const comparison = currentTime.diff(moment(forecastTime.time)); + if (0 < comparison && comparison < closestTimeInPast) { + closestTimeInPast = comparison; + forecast = forecastTime; + } + } + const forecastXHours = this.getForecastForXHoursFrom(forecast.data); + forecast.weatherType = this.convertWeatherType(forecastXHours.summary.symbol_code, forecast.time); + forecast.precipitation = forecastXHours.details?.precipitation_amount; + forecast.minTemperature = forecastXHours.details?.precipitation_amount_min; + forecast.maxTemperature = forecastXHours.details?.precipitation_amount_max; + return this.getWeatherDataFrom(forecast, stellarData, weatherData.properties.meta.units); + }, + + getWeatherData() { + return new Promise((resolve, reject) => { + this.weatherDataQueue.push({ resolve, reject }); + this.getWeatherDataSynchrounous(); + }); + }, + + weatherDataQueue: [], + waitingForWeatherData: false, + // Must be fetched synchrounously to give web request time to populate cache and avoid duplicate calls to the API + getWeatherDataSynchrounous() { + if (this.weatherDataQueue.length < 1) { + return; + } + if (!this.waitingForWeatherData) { + this.waitingForWeatherData = true; + let { resolve, reject } = this.weatherDataQueue.shift(); + + let weatherData = this.getWeatherDataFromCache(); + if (this.weatherDataIsValid(weatherData)) { + this.waitingForWeatherData = false; + this.getWeatherDataSynchrounous(); + Log.debug("Weather data found in cache."); + resolve(weatherData); + } else { + this.getWeatherDataFromYr(weatherData?.downloadedAt) + .then((weatherData) => { + Log.debug("Got weather data from yr."); + if (weatherData) { + this.cacheWeatherData(weatherData); + } else { + //Undefined if unchanged + weatherData = this.getWeatherDataFromCache(); + } + resolve(weatherData); + }) + .catch((err) => { + Log.error(err); + reject("Unable to get weather data from Yr."); + }) + .finally(() => { + this.waitingForWeatherData = false; + this.getWeatherDataSynchrounous(); + }); + } + } + }, + + weatherDataIsValid(weatherData) { + return ( + weatherData && + weatherData.timeout && + 0 < moment(weatherData.timeout).diff(moment()) && + (!weatherData.geometry || !weatherData.geometry.coordinates || !weatherData.geometry.coordinates.length < 2 || (weatherData.geometry.coordinates[0] === this.config.lat && weatherData.geometry.coordinates[1] === this.config.lon)) + ); + }, + + getWeatherDataFromCache() { + let weatherData = undefined; + if (typeof Storage !== "undefined") { + weatherData = localStorage.getItem("weatherData"); + if (weatherData) { + return JSON.parse(weatherData); + } else { + return undefined; + } + } else { + //local storage unavailable + return this.cache?.weatherData; + } + }, + + getWeatherDataFromYr(currentDataFetchedAt) { + const requestHeaders = [{ name: "Accept", value: "application/json" }]; + if (currentDataFetchedAt) { + requestHeaders.push({ name: "If-Modified-Since", value: currentDataFetchedAt }); + } + + const expectedResponseHeaders = ["expires", "date"]; + + return this.fetchData(this.getForecastUrl(), "json", requestHeaders, expectedResponseHeaders) + .then((data) => { + if (!data || !data.headers) return data; + data.timeout = data.headers.find((header) => header.name === "expires").value; + data.downloadedAt = data.headers.find((header) => header.name === "date").value; + data.headers = undefined; + return data; + }) + .catch((err) => { + Log.error("Could not load weather data.", err); + throw new Error(err); + }); + }, + + getForecastUrl() { + if (!this.config.lat) { + Log.error("Latitude not provided."); + throw new Error("Latitude not provided."); + } + if (!this.config.lon) { + Log.error("Longitude not provided."); + throw new Error("Longitude not provided."); + } + + const lat = this.config.lat.toString(); + const lon = this.config.lon.toString(); + const altitude = this.config.altitude ?? 0; + + if (lat.includes(".") && lat.split(".")[1].length > 4) { + Log.error("Latitude is too specific. Do not use more than four decimals."); + throw new Error("Latitude too specific."); + } + if (lon.includes(".") && lon.split(".")[1].length > 4) { + Log.error("Longitude is too specific. Do not use more than four decimals."); + throw new Error("Longitude too specific."); + } + + return `${this.config.apiBase}/locationforecast/2.0/complete?&altitude=${altitude}&lat=${lat}&lon=${lon}`; + }, + + cacheWeatherData(weatherData) { + if (typeof Storage !== "undefined") { + //local storage available + localStorage.setItem("weatherData", JSON.stringify(weatherData)); + } else { + //local storage unavailable + this.cache.weatherData = weatherData; + } + }, + + getAuthenticationString() { + if (!this.config.authenticationEmail) throw new Error("Authentication email not provided."); + return `${this.config.applicaitionName} ${this.config.authenticationEmail}`; + }, + + getStellarData() { + return new Promise((resolve, reject) => { + this.stellarDataQueue.push({ resolve, reject }); + this.getStellarDataSynchrounous(); + }); + }, + + stellarDataQueue: [], + waitingForStellarData: false, + // Must be fetched synchrounously to give web request time to populate cache and avoid duplicate calls to the API. + getStellarDataSynchrounous() { + if (this.stellarDataQueue.length < 1) { + return; + } + if (!this.waitingForStellarData) { + this.waitingForStellarData = true; + let { resolve, reject } = this.stellarDataQueue.shift(); + + let stellarData = this.getStellarDataFromCache(); + const today = moment().format("YYYY-MM-DD"); + const tomorrow = moment().add(1, "days").format("YYYY-MM-DD"); + if (stellarData && stellarData.today && stellarData.today.date === today && stellarData.tomorrow && stellarData.tomorrow.date === tomorrow && this.coordinatesAreCorrect(stellarData.today, stellarData.tomorrow)) { + Log.debug("Stellar data found in cache."); + this.waitingForStellarData = false; + this.getStellarDataSynchrounous(); + resolve(stellarData); + } else if (stellarData && stellarData.tomorrow && stellarData.tomorrow.date === today && this.coordinatesAreCorrect(stellarData.tomorrow)) { + Log.debug("stellar data for today found in cache, but not for tomorrow."); + stellarData.today = stellarData.tomorrow; + this.getStellarDataFromYr(tomorrow) + .then((data) => { + if (data) { + data.date = tomorrow; + stellarData.tomorrow = data; + this.cacheStellarData(stellarData); + resolve(stellarData); + } else { + reject("No stellar data returned from Yr for " + tomorrow); + } + }) + .catch((err) => { + Log.error(err); + reject("Unable to get stellar data from Yr for " + tomorrow); + }) + .finally(() => { + this.waitingForStellarData = false; + this.getStellarDataSynchrounous(); + }); + } else { + this.getStellarDataFromYr(today, 2) + .then((stellarData) => { + if (stellarData) { + stellarData = { + today: stellarData + }; + stellarData.tomorrow = Object.assign({}, stellarData.today); + stellarData.today.date = today; + stellarData.tomorrow.date = tomorrow; + this.cacheStellarData(stellarData); + resolve(stellarData); + } else { + Log.error("Something went wrong when fetching stellar data. Responses: " + stellarData); + reject(stellarData); + } + }) + .catch((err) => { + Log.error(err); + reject("Unable to get stellar data from Yr."); + }) + .finally(() => { + this.waitingForStellarData = false; + this.getStellarDataSynchrounous(); + }); + } + } + }, + + coordinatesAreCorrect(todayData, tomorrowData) { + if (tomorrowData) + return ( + !todayData.location || + (Math.abs(parseFloat(this.config.lat) - parseFloat(todayData.location.latitude)) < 0.1 && + Math.abs(parseFloat(this.config.lon) - parseFloat(todayData.location.longitude)) < 0.1 && + (!tomorrowData.location || (Math.abs(parseFloat(this.config.lat) - parseFloat(tomorrowData.location.latitude)) < 0.1 && Math.abs(parseFloat(this.config.lon) - parseFloat(tomorrowData.location.longitude)) < 0.1))) + ); + return !todayData.location || (Math.abs(parseFloat(this.config.lat) - parseFloat(todayData.location.latitude)) < 0.1 && Math.abs(parseFloat(this.config.lon) - parseFloat(todayData.location.longitude)) < 0.1); + }, + + getStellarDataFromCache() { + let stellarData = undefined; + if (typeof Storage !== "undefined") { + stellarData = localStorage.getItem("stellarData"); + if (stellarData) { + return JSON.parse(stellarData); + } else { + return undefined; + } + } else { + //local storage unavailable + return this.cache?.stellarData; + } + }, + + getStellarDataFromYr(date, days = 1) { + const requestHeaders = [{ name: "Accept", value: "application/json" }]; + return this.fetchData(this.getStellarDatatUrl(date, days), "json", requestHeaders) + .then((data) => { + Log.debug("Got stellar data from yr."); + return data; + }) + .catch((err) => { + Log.error("Could not load weather data.", err); + throw new Error(err); + }); + }, + + getStellarDatatUrl(date, days) { + if (!this.config.lat) { + Log.error("Latitude not provided."); + throw new Error("Latitude not provided."); + } + if (!this.config.lon) { + Log.error("Longitude not provided."); + throw new Error("Longitude not provided."); + } + + const lat = this.config.lat.toString(); + const lon = this.config.lon.toString(); + const altitude = this.config.altitude ?? 0; + + if (lat.includes(".") && lat.split(".")[1].length > 4) { + Log.error("Latitude is too specific. Do not use more than four decimals."); + throw new Error("Latitude too specific."); + } + if (lon.includes(".") && lon.split(".")[1].length > 4) { + Log.error("Longitude is too specific. Do not use more than four decimals."); + throw new Error("Longitude too specific."); + } + + let utcOffset = moment().utcOffset() / 60; + let utcOffsetPrefix = "%2B"; + if (utcOffset < 0) { + utcOffsetPrefix = "-"; + } + utcOffset = Math.abs(utcOffset); + let minutes = "00"; + if (utcOffset % 1 !== 0) { + minutes = "30"; + } + let hours = Math.floor(utcOffset).toString(); + if (hours.length < 2) { + hours = `0${hours}`; + } + + return `${this.config.apiBase}/sunrise/2.0/.json?date=${date}&days=${days}&height=${altitude}&lat=${lat}&lon=${lon}&offset=${utcOffsetPrefix}${hours}%3A${minutes}`; + }, + + cacheStellarData(data) { + if (typeof Storage !== "undefined") { + //local storage available + localStorage.setItem("stellarData", JSON.stringify(data)); + } else { + //local storage unavailable + this.cache.stellarData = data; + } + }, + + getWeatherDataFrom(forecast, stellarData, units) { + const weather = new WeatherObject(this.config.units, this.config.tempUnits, this.config.windUnits, this.config.useKmh); + const stellarTimesToday = stellarData?.today ? this.getStellarTimesFrom(stellarData.today, moment().format("YYYY-MM-DD")) : undefined; + const stellarTimesTomorrow = stellarData?.tomorrow ? this.getStellarTimesFrom(stellarData.tomorrow, moment().add(1, "days").format("YYYY-MM-DD")) : undefined; + + weather.date = moment(forecast.time); + weather.windSpeed = forecast.data.instant.details.wind_speed; + weather.windDirection = (forecast.data.instant.details.wind_from_direction + 180) % 360; + weather.temperature = forecast.data.instant.details.air_temperature; + weather.minTemperature = forecast.minTemperature; + weather.maxTemperature = forecast.maxTemperature; + weather.weatherType = forecast.weatherType; + weather.humidity = forecast.data.instant.details.relative_humidity; + weather.precipitation = forecast.precipitation; + weather.precipitationUnits = units.precipitation_amount; + + if (stellarTimesToday) { + weather.sunset = moment(stellarTimesToday.sunset.time); + weather.sunrise = weather.sunset < moment() && stellarTimesTomorrow ? moment(stellarTimesTomorrow.sunrise.time) : moment(stellarTimesToday.sunrise.time); + } + + return weather; + }, + + convertWeatherType(weatherType, weatherTime) { + const weatherHour = moment(weatherTime).format("HH"); + + const weatherTypes = { + clearsky_day: "day-sunny", + clearsky_night: "night-clear", + clearsky_polartwilight: weatherHour < 14 ? "sunrise" : "sunset", + cloudy: "cloudy", + fair_day: "day-sunny-overcast", + fair_night: "night-alt-partly-cloudy", + fair_polartwilight: "day-sunny-overcast", + fog: "fog", + heavyrain: "rain", // Possibly raindrops or raindrop + heavyrainandthunder: "thunderstorm", + heavyrainshowers_day: "day-rain", + heavyrainshowers_night: "night-alt-rain", + heavyrainshowers_polartwilight: "day-rain", + heavyrainshowersandthunder_day: "day-thunderstorm", + heavyrainshowersandthunder_night: "night-alt-thunderstorm", + heavyrainshowersandthunder_polartwilight: "day-thunderstorm", + heavysleet: "sleet", + heavysleetandthunder: "day-sleet-storm", + heavysleetshowers_day: "day-sleet", + heavysleetshowers_night: "night-alt-sleet", + heavysleetshowers_polartwilight: "day-sleet", + heavysleetshowersandthunder_day: "day-sleet-storm", + heavysleetshowersandthunder_night: "night-alt-sleet-storm", + heavysleetshowersandthunder_polartwilight: "day-sleet-storm", + heavysnow: "snow-wind", + heavysnowandthunder: "day-snow-thunderstorm", + heavysnowshowers_day: "day-snow-wind", + heavysnowshowers_night: "night-alt-snow-wind", + heavysnowshowers_polartwilight: "day-snow-wind", + heavysnowshowersandthunder_day: "day-snow-thunderstorm", + heavysnowshowersandthunder_night: "night-alt-snow-thunderstorm", + heavysnowshowersandthunder_polartwilight: "day-snow-thunderstorm", + lightrain: "rain-mix", + lightrainandthunder: "thunderstorm", + lightrainshowers_day: "day-rain-mix", + lightrainshowers_night: "night-alt-rain-mix", + lightrainshowers_polartwilight: "day-rain-mix", + lightrainshowersandthunder_day: "thunderstorm", + lightrainshowersandthunder_night: "thunderstorm", + lightrainshowersandthunder_polartwilight: "thunderstorm", + lightsleet: "day-sleet", + lightsleetandthunder: "day-sleet-storm", + lightsleetshowers_day: "day-sleet", + lightsleetshowers_night: "night-alt-sleet", + lightsleetshowers_polartwilight: "day-sleet", + lightsnow: "snowflake-cold", + lightsnowandthunder: "day-snow-thunderstorm", + lightsnowshowers_day: "day-snow-wind", + lightsnowshowers_night: "night-alt-snow-wind", + lightsnowshowers_polartwilight: "day-snow-wind", + lightssleetshowersandthunder_day: "day-sleet-storm", + lightssleetshowersandthunder_night: "night-alt-sleet-storm", + lightssleetshowersandthunder_polartwilight: "day-sleet-storm", + lightssnowshowersandthunder_day: "day-snow-thunderstorm", + lightssnowshowersandthunder_night: "night-alt-snow-thunderstorm", + lightssnowshowersandthunder_polartwilight: "day-snow-thunderstorm", + partlycloudy_day: "day-cloudy", + partlycloudy_night: "night-alt-cloudy", + partlycloudy_polartwilight: "day-cloudy", + rain: "rain", + rainandthunder: "thunderstorm", + rainshowers_day: "day-rain", + rainshowers_night: "night-alt-rain", + rainshowers_polartwilight: "day-rain", + rainshowersandthunder_day: "thunderstorm", + rainshowersandthunder_night: "lightning", + rainshowersandthunder_polartwilight: "thunderstorm", + sleet: "sleet", + sleetandthunder: "day-sleet-storm", + sleetshowers_day: "day-sleet", + sleetshowers_night: "night-alt-sleet", + sleetshowers_polartwilight: "day-sleet", + sleetshowersandthunder_day: "day-sleet-storm", + sleetshowersandthunder_night: "night-alt-sleet-storm", + sleetshowersandthunder_polartwilight: "day-sleet-storm", + snow: "snowflake-cold", + snowandthunder: "lightning", + snowshowers_day: "day-snow-wind", + snowshowers_night: "night-alt-snow-wind", + snowshowers_polartwilight: "day-snow-wind", + snowshowersandthunder_day: "day-snow-thunderstorm", + snowshowersandthunder_night: "night-alt-snow-thunderstorm", + snowshowersandthunder_polartwilight: "day-snow-thunderstorm" + }; + + return weatherTypes.hasOwnProperty(weatherType) ? weatherTypes[weatherType] : null; + }, + + getStellarTimesFrom(stellarData, date) { + for (const time of stellarData.location.time) { + if (time.date === date) { + return time; + } + } + return undefined; + }, + + getForecastForXHoursFrom(weather) { + if (this.config.currentForecastHours === 1) { + if (weather.next_1_hours) { + return weather.next_1_hours; + } else if (weather.next_6_hours) { + return weather.next_6_hours; + } else { + return weather.next_12_hours; + } + } else if (this.config.currentForecastHours === 6) { + if (weather.next_6_hours) { + return weather.next_6_hours; + } else if (weather.next_12_hours) { + return weather.next_12_hours; + } else { + return weather.next_1_hours; + } + } else { + if (weather.next_12_hours) { + return weather.next_12_hours; + } else if (weather.next_6_hours) { + return weather.next_6_hours; + } else { + return weather.next_1_hours; + } + } + }, + + fetchWeatherHourly() { + this.getWeatherForecast("hourly") + .then((forecast) => { + this.setWeatherHourly(forecast); + this.updateAvailable(); + }) + .catch((error) => { + Log.error(error); + throw new Error(error); + }); + }, + + async getWeatherForecast(type) { + const getRequests = [this.getWeatherData(), this.getStellarData()]; + const [weatherData, stellarData] = await Promise.all(getRequests); + if (!weatherData.properties.timeseries || !weatherData.properties.timeseries[0]) { + Log.error("No weather data available."); + return; + } + if (!stellarData) { + Log.warn("No stelar data available."); + } + let forecasts; + switch (type) { + case "hourly": + forecasts = this.getHourlyForecastFrom(weatherData); + break; + case "daily": + default: + forecasts = this.getDailyForecastFrom(weatherData); + break; + } + const series = []; + for (const forecast of forecasts) { + series.push(this.getWeatherDataFrom(forecast, stellarData, weatherData.properties.meta.units)); + } + return series; + }, + + getHourlyForecastFrom(weatherData) { + const series = []; + + for (const forecast of weatherData.properties.timeseries) { + forecast.symbol = forecast.data.next_1_hours?.summary?.symbol_code; + forecast.precipitation = forecast.data.next_1_hours?.details?.precipitation_amount; + forecast.minTemperature = forecast.data.next_1_hours?.details?.air_temperature_min; + forecast.maxTemperature = forecast.data.next_1_hours?.details?.air_temperature_max; + forecast.weatherType = this.convertWeatherType(forecast.symbol, forecast.time); + series.push(forecast); + } + return series; + }, + + getDailyForecastFrom(weatherData) { + const series = []; + + const days = weatherData.properties.timeseries.reduce(function (days, forecast) { + const date = moment(forecast.time).format("YYYY-MM-DD"); + days[date] = days[date] || []; + days[date].push(forecast); + return days; + }, Object.create(null)); + + Object.keys(days).forEach(function (time, index) { + let precipitation_amount_min = undefined; + let precipitation_amount_max = undefined; + + //Default to first entry + let forecast = days[time][0]; + forecast.symbol = forecast.data.next_12_hours?.summary?.symbol_code; + forecast.precipitation = forecast.data.next_12_hours?.details?.precipitation_amount; + + //Coming days + let forecastDiffToEight = undefined; + for (const timeseries of days[time]) { + if (!timeseries.data.next_6_hours) continue; //next_6_hours has the most data + + if (!precipitation_amount_min || timeseries.data.next_6_hours.details.air_temperature_min < precipitation_amount_min) precipitation_amount_min = timeseries.data.next_6_hours.details.air_temperature_min; + if (!precipitation_amount_max || precipitation_amount_max < timeseries.data.next_6_hours.details.air_temperature_max) precipitation_amount_max = timeseries.data.next_6_hours.details.air_temperature_max; + + let closestTime = Math.abs(moment(timeseries.time).local().set({ hour: 8, minute: 0, second: 0, millisecond: 0 }).diff(moment(timeseries.time).local())); + if ((forecastDiffToEight === undefined || closestTime < forecastDiffToEight) && timeseries.data.next_12_hours) { + forecastDiffToEight = closestTime; + forecast = timeseries; + } + } + const forecastXHours = forecast.data.next_12_hours ?? forecast.data.next_6_hours ?? forecast.data.next_1_hours; + forecast.symbol = forecastXHours.summary?.symbol_code; + forecast.precipitation = forecastXHours.details?.precipitation_amount; + forecast.minTemperature = precipitation_amount_min; + forecast.maxTemperature = precipitation_amount_max; + + series.push(forecast); + }); + for (const forecast of series) { + forecast.weatherType = this.convertWeatherType(forecast.symbol, forecast.time); + } + return series; + }, + + fetchWeatherForecast() { + this.getWeatherForecast("daily") + .then((forecast) => { + this.setWeatherForecast(forecast); + this.updateAvailable(); + }) + .catch((error) => { + Log.error(error); + throw new Error(error); + }); + } +}); diff --git a/modules/default/weather/weatherprovider.js b/modules/default/weather/weatherprovider.js index bf1deee2d4..b780950bb3 100644 --- a/modules/default/weather/weatherprovider.js +++ b/modules/default/weather/weatherprovider.js @@ -111,23 +111,17 @@ const WeatherProvider = Class.extend({ this.delegate.updateAvailable(this); }, - getCorsUrl: function () { - if (this.config.mockData || typeof this.config.useCorsProxy === "undefined" || !this.config.useCorsProxy) { - return ""; - } else { - return location.protocol + "//" + location.host + "/cors?url="; - } - }, - /** * A convenience function to make requests. * * @param {string} url the url to fetch from * @param {string} type what contenttype to expect in the response, can be "json" or "xml" + * @param {Array.<{name: string, value:string}>} requestHeaders the HTTP headers to send + * @param {Array.} expectedResponseHeaders the expected HTTP headers to recieve * @returns {Promise} resolved when the fetch is done */ - fetchData: async function (url, type = "json") { - url = this.getCorsUrl() + url; + fetchData: async function (url, type = "json", requestHeaders = undefined, expectedResponseHeaders = undefined) { + url = this.getCorsUrl(url, requestHeaders, expectedResponseHeaders); const mockData = this.config.mockData; if (mockData) { const data = mockData.substring(1, mockData.length - 1); @@ -135,12 +129,114 @@ const WeatherProvider = Class.extend({ } else { const response = await fetch(url); const data = await response.text(); + if (type === "xml") { return new DOMParser().parseFromString(data, "text/html"); } else { - return JSON.parse(data); + if (!data || !data.length > 0) return undefined; + + const dataResponse = JSON.parse(data); + if (!dataResponse.headers) { + dataResponse.headers = this.getHeadersFromResponse(expectedResponseHeaders, response); + } + return dataResponse; + } + } + }, + + /** + * Gets a URL that will be used when calling the CORS-method on the server. + * + * @param {string} url the url to fetch from + * @param {Array.<{name: string, value:string}>} requestHeaders the HTTP headers to send + * @param {Array.} expectedResponseHeaders the expected HTTP headers to recieve + * @returns {string} to be used as URL when calling CORS-method on server. + */ + getCorsUrl: function (url, requestHeaders, expectedResponseHeaders) { + if (this.config.mockData || typeof this.config.useCorsProxy === "undefined" || !this.config.useCorsProxy) { + return ""; + } else if (!url || url.length < 1) { + throw new Error(`Invalid URL: ${url}`); + } else { + let corsUrl = `${location.protocol}//${location.host}/cors?`; + + const requestHeaderString = this.getRequestHeaderString(requestHeaders); + if (requestHeaderString) corsUrl = `${corsUrl}sendheaders=${requestHeaderString}`; + + const expectedResponseHeadersString = this.getExpectedResponseHeadersString(expectedResponseHeaders); + if (requestHeaderString && expectedResponseHeadersString) { + corsUrl = `${corsUrl}&expectedheaders=${expectedResponseHeadersString}`; + } else if (expectedResponseHeadersString) { + corsUrl = `${corsUrl}expectedheaders=${expectedResponseHeadersString}`; + } + + if (requestHeaderString || expectedResponseHeadersString) { + return `${corsUrl}&url=${url}`; } + return `${corsUrl}url=${url}`; } + }, + + /** + * Gets the part of the CORS URL that represents the HTTP headers to send. + * + * @param {Array.<{name: string, value:string}>} requestHeaders the HTTP headers to send + * @returns {string} to be used as request-headers component in CORS URL. + */ + getRequestHeaderString: function (requestHeaders) { + let requestHeaderString = ""; + if (requestHeaders) { + for (const header of requestHeaders) { + if (requestHeaderString.length === 0) { + requestHeaderString = `${header.name}:${encodeURIComponent(header.value)}`; + } else { + requestHeaderString = `${requestHeaderString},${header.name}:${encodeURIComponent(header.value)}`; + } + } + return requestHeaderString; + } + return undefined; + }, + + /** + * Gets the part of the CORS URL that represents the expected HTTP headers to recieve. + * + * @param {Array.} expectedResponseHeaders the expected HTTP headers to recieve + * @returns {string} to be used as the expected HTTP-headers component in CORS URL. + */ + getExpectedResponseHeadersString: function (expectedResponseHeaders) { + let expectedResponseHeadersString = ""; + if (expectedResponseHeaders) { + for (const header of expectedResponseHeaders) { + if (expectedResponseHeadersString.length === 0) { + expectedResponseHeadersString = `${header}`; + } else { + expectedResponseHeadersString = `${expectedResponseHeadersString},${header}`; + } + } + return expectedResponseHeaders; + } + return undefined; + }, + + /** + * Gets the values for the expected headers from the response. + * + * @param {Array.} expectedResponseHeaders the expected HTTP headers to recieve + * @param {Response} response the HTTP response + * @returns {string} to be used as the expected HTTP-headers component in CORS URL. + */ + getHeadersFromResponse(expectedResponseHeaders, response) { + const responseHeaders = []; + + if (expectedResponseHeaders) { + for (const header of expectedResponseHeaders) { + const headerValue = response.headers.get(header); + responseHeaders.push({ name: header, value: headerValue }); + } + } + + return responseHeaders; } }); From 248c9916c1e62b663460d26c39966b61b8821f59 Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Wed, 2 Nov 2022 19:52:43 +0100 Subject: [PATCH 2/9] Synchronization for fetching data from yr. If a user has multiple instances of the same weather-module, this may lead to multiple calls to the API. This local storage-based synchronization is an attempt to avoid that. --- modules/default/weather/providers/yr.js | 287 ++++++++++++------------ 1 file changed, 138 insertions(+), 149 deletions(-) diff --git a/modules/default/weather/providers/yr.js b/modules/default/weather/providers/yr.js index ffc8685f85..19f04911fd 100644 --- a/modules/default/weather/providers/yr.js +++ b/modules/default/weather/providers/yr.js @@ -22,13 +22,13 @@ WeatherProvider.register("yr", { currentForecastHours: 1 //1, 6 or 12 }, - //Backup cache if local storage does not work - cache: { - weatherData: undefined, - stellarData: { - today: undefined, - tomorrow: undefined + start() { + if (typeof Storage === "undefined") { + //local storage unavailable + Log.error("The Yr weather provider requires local storage."); + throw new Error("Local storage not available"); } + Log.info(`Weather provider: ${this.providerName} started.`); }, fetchCurrentWeather() { @@ -73,49 +73,56 @@ WeatherProvider.register("yr", { getWeatherData() { return new Promise((resolve, reject) => { - this.weatherDataQueue.push({ resolve, reject }); - this.getWeatherDataSynchrounous(); + // If a user has several Yr-modules, for instance one current and one forecast, the API calls must be synchronized across classes. + // This is to avoid multiple similar calls to the API. + let shouldWait = localStorage.getItem("yrIsFetchingWeatherData"); + if (shouldWait) { + const checkForGo = setInterval(function () { + shouldWait = localStorage.getItem("yrIsFetchingWeatherData"); + }, 100); + setTimeout(function () { + clearInterval(checkForGo); + shouldWait = false; + }, 5000); //Assume other fetch finished but failed to remove lock + const attemptFetchWeather = setInterval(() => { + if (!shouldWait) { + clearInterval(attemptFetchWeather); + this.getWeatherDataFromYrOrCache(resolve, reject); + } + }, 100); + } else { + this.getWeatherDataFromYrOrCache(resolve, reject); + } }); }, - weatherDataQueue: [], - waitingForWeatherData: false, - // Must be fetched synchrounously to give web request time to populate cache and avoid duplicate calls to the API - getWeatherDataSynchrounous() { - if (this.weatherDataQueue.length < 1) { - return; - } - if (!this.waitingForWeatherData) { - this.waitingForWeatherData = true; - let { resolve, reject } = this.weatherDataQueue.shift(); - - let weatherData = this.getWeatherDataFromCache(); - if (this.weatherDataIsValid(weatherData)) { - this.waitingForWeatherData = false; - this.getWeatherDataSynchrounous(); - Log.debug("Weather data found in cache."); - resolve(weatherData); - } else { - this.getWeatherDataFromYr(weatherData?.downloadedAt) - .then((weatherData) => { - Log.debug("Got weather data from yr."); - if (weatherData) { - this.cacheWeatherData(weatherData); - } else { - //Undefined if unchanged - weatherData = this.getWeatherDataFromCache(); - } - resolve(weatherData); - }) - .catch((err) => { - Log.error(err); - reject("Unable to get weather data from Yr."); - }) - .finally(() => { - this.waitingForWeatherData = false; - this.getWeatherDataSynchrounous(); - }); - } + getWeatherDataFromYrOrCache(resolve, reject) { + localStorage.setItem("yrIsFetchingWeatherData", "true"); + + let weatherData = this.getWeatherDataFromCache(); + if (this.weatherDataIsValid(weatherData)) { + localStorage.removeItem("yrIsFetchingWeatherData"); + Log.debug("Weather data found in cache."); + resolve(weatherData); + } else { + this.getWeatherDataFromYr(weatherData?.downloadedAt) + .then((weatherData) => { + Log.debug("Got weather data from yr."); + if (weatherData) { + this.cacheWeatherData(weatherData); + } else { + //Undefined if unchanged + weatherData = this.getWeatherDataFromCache(); + } + resolve(weatherData); + }) + .catch((err) => { + Log.error(err); + reject("Unable to get weather data from Yr."); + }) + .finally(() => { + localStorage.removeItem("yrIsFetchingWeatherData"); + }); } }, @@ -129,17 +136,11 @@ WeatherProvider.register("yr", { }, getWeatherDataFromCache() { - let weatherData = undefined; - if (typeof Storage !== "undefined") { - weatherData = localStorage.getItem("weatherData"); - if (weatherData) { - return JSON.parse(weatherData); - } else { - return undefined; - } + const weatherData = localStorage.getItem("weatherData"); + if (weatherData) { + return JSON.parse(weatherData); } else { - //local storage unavailable - return this.cache?.weatherData; + return undefined; } }, @@ -192,13 +193,7 @@ WeatherProvider.register("yr", { }, cacheWeatherData(weatherData) { - if (typeof Storage !== "undefined") { - //local storage available - localStorage.setItem("weatherData", JSON.stringify(weatherData)); - } else { - //local storage unavailable - this.cache.weatherData = weatherData; - } + localStorage.setItem("weatherData", JSON.stringify(weatherData)); }, getAuthenticationString() { @@ -207,79 +202,85 @@ WeatherProvider.register("yr", { }, getStellarData() { + // If a user has several Yr-modules, for instance one current and one forecast, the API calls must be synchronized across classes. + // This is to avoid multiple similar calls to the API. return new Promise((resolve, reject) => { - this.stellarDataQueue.push({ resolve, reject }); - this.getStellarDataSynchrounous(); + let shouldWait = localStorage.getItem("yrIsFetchingStellarData"); + if (shouldWait) { + const checkForGo = setInterval(function () { + shouldWait = localStorage.getItem("yrIsFetchingStellarData"); + }, 100); + setTimeout(function () { + clearInterval(checkForGo); + shouldWait = false; + }, 5000); //Assume other fetch finished but failed to remove lock + const attemptFetchWeather = setInterval(() => { + if (!shouldWait) { + clearInterval(attemptFetchWeather); + this.getStellarDataFromYrOrCache(resolve, reject); + } + }, 100); + } else { + this.getStellarDataFromYrOrCache(resolve, reject); + } }); }, - stellarDataQueue: [], - waitingForStellarData: false, - // Must be fetched synchrounously to give web request time to populate cache and avoid duplicate calls to the API. - getStellarDataSynchrounous() { - if (this.stellarDataQueue.length < 1) { - return; - } - if (!this.waitingForStellarData) { - this.waitingForStellarData = true; - let { resolve, reject } = this.stellarDataQueue.shift(); - - let stellarData = this.getStellarDataFromCache(); - const today = moment().format("YYYY-MM-DD"); - const tomorrow = moment().add(1, "days").format("YYYY-MM-DD"); - if (stellarData && stellarData.today && stellarData.today.date === today && stellarData.tomorrow && stellarData.tomorrow.date === tomorrow && this.coordinatesAreCorrect(stellarData.today, stellarData.tomorrow)) { - Log.debug("Stellar data found in cache."); - this.waitingForStellarData = false; - this.getStellarDataSynchrounous(); - resolve(stellarData); - } else if (stellarData && stellarData.tomorrow && stellarData.tomorrow.date === today && this.coordinatesAreCorrect(stellarData.tomorrow)) { - Log.debug("stellar data for today found in cache, but not for tomorrow."); - stellarData.today = stellarData.tomorrow; - this.getStellarDataFromYr(tomorrow) - .then((data) => { - if (data) { - data.date = tomorrow; - stellarData.tomorrow = data; - this.cacheStellarData(stellarData); - resolve(stellarData); - } else { - reject("No stellar data returned from Yr for " + tomorrow); - } - }) - .catch((err) => { - Log.error(err); - reject("Unable to get stellar data from Yr for " + tomorrow); - }) - .finally(() => { - this.waitingForStellarData = false; - this.getStellarDataSynchrounous(); - }); - } else { - this.getStellarDataFromYr(today, 2) - .then((stellarData) => { - if (stellarData) { - stellarData = { - today: stellarData - }; - stellarData.tomorrow = Object.assign({}, stellarData.today); - stellarData.today.date = today; - stellarData.tomorrow.date = tomorrow; - this.cacheStellarData(stellarData); - resolve(stellarData); - } else { - Log.error("Something went wrong when fetching stellar data. Responses: " + stellarData); - reject(stellarData); - } - }) - .catch((err) => { - Log.error(err); - reject("Unable to get stellar data from Yr."); - }) - .finally(() => { - this.waitingForStellarData = false; - this.getStellarDataSynchrounous(); - }); - } + getStellarDataFromYrOrCache(resolve, reject) { + localStorage.setItem("yrIsFetchingStellarData", "true"); + + let stellarData = this.getStellarDataFromCache(); + const today = moment().format("YYYY-MM-DD"); + const tomorrow = moment().add(1, "days").format("YYYY-MM-DD"); + if (stellarData && stellarData.today && stellarData.today.date === today && stellarData.tomorrow && stellarData.tomorrow.date === tomorrow && this.coordinatesAreCorrect(stellarData.today, stellarData.tomorrow)) { + Log.debug("Stellar data found in cache."); + localStorage.removeItem("yrIsFetchingStellarData"); + resolve(stellarData); + } else if (stellarData && stellarData.tomorrow && stellarData.tomorrow.date === today && this.coordinatesAreCorrect(stellarData.tomorrow)) { + Log.debug("stellar data for today found in cache, but not for tomorrow."); + stellarData.today = stellarData.tomorrow; + this.getStellarDataFromYr(tomorrow) + .then((data) => { + if (data) { + data.date = tomorrow; + stellarData.tomorrow = data; + this.cacheStellarData(stellarData); + resolve(stellarData); + } else { + reject("No stellar data returned from Yr for " + tomorrow); + } + }) + .catch((err) => { + Log.error(err); + reject("Unable to get stellar data from Yr for " + tomorrow); + }) + .finally(() => { + localStorage.removeItem("yrIsFetchingStellarData"); + }); + } else { + this.getStellarDataFromYr(today, 2) + .then((stellarData) => { + if (stellarData) { + stellarData = { + today: stellarData + }; + stellarData.tomorrow = Object.assign({}, stellarData.today); + stellarData.today.date = today; + stellarData.tomorrow.date = tomorrow; + this.cacheStellarData(stellarData); + resolve(stellarData); + } else { + Log.error("Something went wrong when fetching stellar data. Responses: " + stellarData); + reject(stellarData); + } + }) + .catch((err) => { + Log.error(err); + reject("Unable to get stellar data from Yr."); + }) + .finally(() => { + localStorage.removeItem("yrIsFetchingStellarData"); + }); } }, @@ -295,17 +296,11 @@ WeatherProvider.register("yr", { }, getStellarDataFromCache() { - let stellarData = undefined; - if (typeof Storage !== "undefined") { - stellarData = localStorage.getItem("stellarData"); - if (stellarData) { - return JSON.parse(stellarData); - } else { - return undefined; - } + const stellarData = localStorage.getItem("stellarData"); + if (stellarData) { + return JSON.parse(stellarData); } else { - //local storage unavailable - return this.cache?.stellarData; + return undefined; } }, @@ -364,13 +359,7 @@ WeatherProvider.register("yr", { }, cacheStellarData(data) { - if (typeof Storage !== "undefined") { - //local storage available - localStorage.setItem("stellarData", JSON.stringify(data)); - } else { - //local storage unavailable - this.cache.stellarData = data; - } + localStorage.setItem("stellarData", JSON.stringify(data)); }, getWeatherDataFrom(forecast, stellarData, units) { From ab9320b0fe2e6917a859ac8f0558a866ec9e80fc Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Wed, 2 Nov 2022 20:23:53 +0100 Subject: [PATCH 3/9] Handles forecast with only instant-details. --- modules/default/weather/providers/yr.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/default/weather/providers/yr.js b/modules/default/weather/providers/yr.js index 19f04911fd..30025b2166 100644 --- a/modules/default/weather/providers/yr.js +++ b/modules/default/weather/providers/yr.js @@ -602,12 +602,14 @@ WeatherProvider.register("yr", { } } const forecastXHours = forecast.data.next_12_hours ?? forecast.data.next_6_hours ?? forecast.data.next_1_hours; - forecast.symbol = forecastXHours.summary?.symbol_code; - forecast.precipitation = forecastXHours.details?.precipitation_amount; - forecast.minTemperature = precipitation_amount_min; - forecast.maxTemperature = precipitation_amount_max; + if (forecastXHours) { + forecast.symbol = forecastXHours.summary?.symbol_code; + forecast.precipitation = forecastXHours.details?.precipitation_amount; + forecast.minTemperature = precipitation_amount_min; + forecast.maxTemperature = precipitation_amount_max; - series.push(forecast); + series.push(forecast); + } }); for (const forecast of series) { forecast.weatherType = this.convertWeatherType(forecast.symbol, forecast.time); From 5be6a6600741f0427f55fe32e73994261687610f Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Thu, 3 Nov 2022 16:47:29 +0100 Subject: [PATCH 4/9] minor fixes --- CHANGELOG.md | 1 - modules/default/weather/providers/yr.js | 16 ++++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4c7badf2c..4e9841b33b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,6 @@ Special thanks to: @rejas, @sdetweil, @MagMar94 - Reworked how weatherproviders handle units (#2849) - Use unix() method for parsing times, fix suntimes on the way (#2950) - Refactor conversion functions into utils class (#2958) - - Support HTTP headers - The `cors`-method in `server.js` now supports sending and recieving HTTP headers. ### Fixed diff --git a/modules/default/weather/providers/yr.js b/modules/default/weather/providers/yr.js index 30025b2166..d2b3e0ad44 100644 --- a/modules/default/weather/providers/yr.js +++ b/modules/default/weather/providers/yr.js @@ -66,8 +66,8 @@ WeatherProvider.register("yr", { const forecastXHours = this.getForecastForXHoursFrom(forecast.data); forecast.weatherType = this.convertWeatherType(forecastXHours.summary.symbol_code, forecast.time); forecast.precipitation = forecastXHours.details?.precipitation_amount; - forecast.minTemperature = forecastXHours.details?.precipitation_amount_min; - forecast.maxTemperature = forecastXHours.details?.precipitation_amount_max; + forecast.minTemperature = forecastXHours.details?.air_temperature_min; + forecast.maxTemperature = forecastXHours.details?.air_temperature_max; return this.getWeatherDataFrom(forecast, stellarData, weatherData.properties.meta.units); }, @@ -579,8 +579,8 @@ WeatherProvider.register("yr", { }, Object.create(null)); Object.keys(days).forEach(function (time, index) { - let precipitation_amount_min = undefined; - let precipitation_amount_max = undefined; + let minTemperature = undefined; + let maxTemperature = undefined; //Default to first entry let forecast = days[time][0]; @@ -592,8 +592,8 @@ WeatherProvider.register("yr", { for (const timeseries of days[time]) { if (!timeseries.data.next_6_hours) continue; //next_6_hours has the most data - if (!precipitation_amount_min || timeseries.data.next_6_hours.details.air_temperature_min < precipitation_amount_min) precipitation_amount_min = timeseries.data.next_6_hours.details.air_temperature_min; - if (!precipitation_amount_max || precipitation_amount_max < timeseries.data.next_6_hours.details.air_temperature_max) precipitation_amount_max = timeseries.data.next_6_hours.details.air_temperature_max; + if (!minTemperature || timeseries.data.next_6_hours.details.air_temperature_min < minTemperature) minTemperature = timeseries.data.next_6_hours.details.air_temperature_min; + if (!maxTemperature || maxTemperature < timeseries.data.next_6_hours.details.air_temperature_max) maxTemperature = timeseries.data.next_6_hours.details.air_temperature_max; let closestTime = Math.abs(moment(timeseries.time).local().set({ hour: 8, minute: 0, second: 0, millisecond: 0 }).diff(moment(timeseries.time).local())); if ((forecastDiffToEight === undefined || closestTime < forecastDiffToEight) && timeseries.data.next_12_hours) { @@ -605,8 +605,8 @@ WeatherProvider.register("yr", { if (forecastXHours) { forecast.symbol = forecastXHours.summary?.symbol_code; forecast.precipitation = forecastXHours.details?.precipitation_amount; - forecast.minTemperature = precipitation_amount_min; - forecast.maxTemperature = precipitation_amount_max; + forecast.minTemperature = minTemperature; + forecast.maxTemperature = maxTemperature; series.push(forecast); } From eff5bd72f1f81e29d9364896018dc1ea260a53d8 Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Sun, 6 Nov 2022 16:58:33 +0100 Subject: [PATCH 5/9] Moved the fetch-logic to a common utils-file that can be used ty other modules. --- modules/default/utils.js | 146 +++++++++++++++++++++ modules/default/weather/weather.js | 2 +- modules/default/weather/weatherprovider.js | 115 +--------------- 3 files changed, 150 insertions(+), 113 deletions(-) create mode 100644 modules/default/utils.js diff --git a/modules/default/utils.js b/modules/default/utils.js new file mode 100644 index 0000000000..266f613f62 --- /dev/null +++ b/modules/default/utils.js @@ -0,0 +1,146 @@ +/** + * A function to make HTTP requests via the server to avoid CORS-errors. + * + * @param {string} url the url to fetch from + * @param {string} type what contenttype to expect in the response, can be "json" or "xml" + * @param {boolean} useCorsProxy A flag to indicate + * @param {Array.<{name: string, value:string}>} requestHeaders the HTTP headers to send + * @param {Array.} expectedResponseHeaders the expected HTTP headers to recieve + * @returns {Promise} resolved when the fetch is done. The response headers is placed in a headers-property (provided the response does not allready contain a headers-property). + */ +async function performWebRequest(url, type = "json", useCorsProxy = false, requestHeaders = undefined, expectedResponseHeaders = undefined) { + const request = {}; + if (useCorsProxy) { + url = getCorsUrl(url, requestHeaders, expectedResponseHeaders); + } else { + request.headers = getHeadersToSend(requestHeaders); + } + const response = await fetch(url, request); + const data = await response.text(); + + if (type === "xml") { + return new DOMParser().parseFromString(data, "text/html"); + } else { + if (!data || !data.length > 0) return undefined; + + const dataResponse = JSON.parse(data); + if (!dataResponse.headers) { + dataResponse.headers = getHeadersFromResponse(expectedResponseHeaders, response); + } + return dataResponse; + } +} + +/** + * Gets a URL that will be used when calling the CORS-method on the server. + * + * @param {string} url the url to fetch from + * @param {Array.<{name: string, value:string}>} requestHeaders the HTTP headers to send + * @param {Array.} expectedResponseHeaders the expected HTTP headers to recieve + * @returns {string} to be used as URL when calling CORS-method on server. + */ +const getCorsUrl = function (url, requestHeaders, expectedResponseHeaders) { + if (!url || url.length < 1) { + throw new Error(`Invalid URL: ${url}`); + } else { + let corsUrl = `${location.protocol}//${location.host}/cors?`; + + const requestHeaderString = getRequestHeaderString(requestHeaders); + if (requestHeaderString) corsUrl = `${corsUrl}sendheaders=${requestHeaderString}`; + + const expectedResponseHeadersString = getExpectedResponseHeadersString(expectedResponseHeaders); + if (requestHeaderString && expectedResponseHeadersString) { + corsUrl = `${corsUrl}&expectedheaders=${expectedResponseHeadersString}`; + } else if (expectedResponseHeadersString) { + corsUrl = `${corsUrl}expectedheaders=${expectedResponseHeadersString}`; + } + + if (requestHeaderString || expectedResponseHeadersString) { + return `${corsUrl}&url=${url}`; + } + return `${corsUrl}url=${url}`; + } +}; + +/** + * Gets the part of the CORS URL that represents the HTTP headers to send. + * + * @param {Array.<{name: string, value:string}>} requestHeaders the HTTP headers to send + * @returns {string} to be used as request-headers component in CORS URL. + */ +const getRequestHeaderString = function (requestHeaders) { + let requestHeaderString = ""; + if (requestHeaders) { + for (const header of requestHeaders) { + if (requestHeaderString.length === 0) { + requestHeaderString = `${header.name}:${encodeURIComponent(header.value)}`; + } else { + requestHeaderString = `${requestHeaderString},${header.name}:${encodeURIComponent(header.value)}`; + } + } + return requestHeaderString; + } + return undefined; +}; + +/** + * Gets headers and values to attatch to the web request. + * + * @param {Array.<{name: string, value:string}>} requestHeaders the HTTP headers to send + * @returns {object} An object specifying name and value of the headers. + */ +const getHeadersToSend = (requestHeaders) => { + const headersToSend = {}; + if (requestHeaders) { + for (const header of requestHeaders) { + headersToSend[header.name] = header.value; + } + } + + return headersToSend; +}; + +/** + * Gets the part of the CORS URL that represents the expected HTTP headers to recieve. + * + * @param {Array.} expectedResponseHeaders the expected HTTP headers to recieve + * @returns {string} to be used as the expected HTTP-headers component in CORS URL. + */ +const getExpectedResponseHeadersString = function (expectedResponseHeaders) { + let expectedResponseHeadersString = ""; + if (expectedResponseHeaders) { + for (const header of expectedResponseHeaders) { + if (expectedResponseHeadersString.length === 0) { + expectedResponseHeadersString = `${header}`; + } else { + expectedResponseHeadersString = `${expectedResponseHeadersString},${header}`; + } + } + return expectedResponseHeaders; + } + return undefined; +}; + +/** + * Gets the values for the expected headers from the response. + * + * @param {Array.} expectedResponseHeaders the expected HTTP headers to recieve + * @param {Response} response the HTTP response + * @returns {string} to be used as the expected HTTP-headers component in CORS URL. + */ +const getHeadersFromResponse = (expectedResponseHeaders, response) => { + const responseHeaders = []; + + if (expectedResponseHeaders) { + for (const header of expectedResponseHeaders) { + const headerValue = response.headers.get(header); + responseHeaders.push({ name: header, value: headerValue }); + } + } + + return responseHeaders; +}; + +module.exports = { + performWebRequest +}; diff --git a/modules/default/weather/weather.js b/modules/default/weather/weather.js index 009b4fb8fd..ab989f96ab 100644 --- a/modules/default/weather/weather.js +++ b/modules/default/weather/weather.js @@ -58,7 +58,7 @@ Module.register("weather", { // Return the scripts that are necessary for the weather module. getScripts: function () { - return ["moment.js", "weatherutils.js", "weatherprovider.js", "weatherobject.js", "suncalc.js", this.file("providers/" + this.config.weatherProvider.toLowerCase() + ".js")]; + return ["moment.js", this.file("../utils.js"), "weatherutils.js", "weatherprovider.js", "weatherobject.js", "suncalc.js", this.file("providers/" + this.config.weatherProvider.toLowerCase() + ".js")]; }, // Override getHeader method. diff --git a/modules/default/weather/weatherprovider.js b/modules/default/weather/weatherprovider.js index b780950bb3..e7bfe5b7d7 100644 --- a/modules/default/weather/weatherprovider.js +++ b/modules/default/weather/weatherprovider.js @@ -1,4 +1,4 @@ -/* global Class */ +/* global Class, performWebRequest */ /* MagicMirror² * Module: Weather @@ -121,122 +121,13 @@ const WeatherProvider = Class.extend({ * @returns {Promise} resolved when the fetch is done */ fetchData: async function (url, type = "json", requestHeaders = undefined, expectedResponseHeaders = undefined) { - url = this.getCorsUrl(url, requestHeaders, expectedResponseHeaders); const mockData = this.config.mockData; if (mockData) { const data = mockData.substring(1, mockData.length - 1); return JSON.parse(data); - } else { - const response = await fetch(url); - const data = await response.text(); - - if (type === "xml") { - return new DOMParser().parseFromString(data, "text/html"); - } else { - if (!data || !data.length > 0) return undefined; - - const dataResponse = JSON.parse(data); - if (!dataResponse.headers) { - dataResponse.headers = this.getHeadersFromResponse(expectedResponseHeaders, response); - } - return dataResponse; - } } - }, - - /** - * Gets a URL that will be used when calling the CORS-method on the server. - * - * @param {string} url the url to fetch from - * @param {Array.<{name: string, value:string}>} requestHeaders the HTTP headers to send - * @param {Array.} expectedResponseHeaders the expected HTTP headers to recieve - * @returns {string} to be used as URL when calling CORS-method on server. - */ - getCorsUrl: function (url, requestHeaders, expectedResponseHeaders) { - if (this.config.mockData || typeof this.config.useCorsProxy === "undefined" || !this.config.useCorsProxy) { - return ""; - } else if (!url || url.length < 1) { - throw new Error(`Invalid URL: ${url}`); - } else { - let corsUrl = `${location.protocol}//${location.host}/cors?`; - - const requestHeaderString = this.getRequestHeaderString(requestHeaders); - if (requestHeaderString) corsUrl = `${corsUrl}sendheaders=${requestHeaderString}`; - - const expectedResponseHeadersString = this.getExpectedResponseHeadersString(expectedResponseHeaders); - if (requestHeaderString && expectedResponseHeadersString) { - corsUrl = `${corsUrl}&expectedheaders=${expectedResponseHeadersString}`; - } else if (expectedResponseHeadersString) { - corsUrl = `${corsUrl}expectedheaders=${expectedResponseHeadersString}`; - } - - if (requestHeaderString || expectedResponseHeadersString) { - return `${corsUrl}&url=${url}`; - } - return `${corsUrl}url=${url}`; - } - }, - - /** - * Gets the part of the CORS URL that represents the HTTP headers to send. - * - * @param {Array.<{name: string, value:string}>} requestHeaders the HTTP headers to send - * @returns {string} to be used as request-headers component in CORS URL. - */ - getRequestHeaderString: function (requestHeaders) { - let requestHeaderString = ""; - if (requestHeaders) { - for (const header of requestHeaders) { - if (requestHeaderString.length === 0) { - requestHeaderString = `${header.name}:${encodeURIComponent(header.value)}`; - } else { - requestHeaderString = `${requestHeaderString},${header.name}:${encodeURIComponent(header.value)}`; - } - } - return requestHeaderString; - } - return undefined; - }, - - /** - * Gets the part of the CORS URL that represents the expected HTTP headers to recieve. - * - * @param {Array.} expectedResponseHeaders the expected HTTP headers to recieve - * @returns {string} to be used as the expected HTTP-headers component in CORS URL. - */ - getExpectedResponseHeadersString: function (expectedResponseHeaders) { - let expectedResponseHeadersString = ""; - if (expectedResponseHeaders) { - for (const header of expectedResponseHeaders) { - if (expectedResponseHeadersString.length === 0) { - expectedResponseHeadersString = `${header}`; - } else { - expectedResponseHeadersString = `${expectedResponseHeadersString},${header}`; - } - } - return expectedResponseHeaders; - } - return undefined; - }, - - /** - * Gets the values for the expected headers from the response. - * - * @param {Array.} expectedResponseHeaders the expected HTTP headers to recieve - * @param {Response} response the HTTP response - * @returns {string} to be used as the expected HTTP-headers component in CORS URL. - */ - getHeadersFromResponse(expectedResponseHeaders, response) { - const responseHeaders = []; - - if (expectedResponseHeaders) { - for (const header of expectedResponseHeaders) { - const headerValue = response.headers.get(header); - responseHeaders.push({ name: header, value: headerValue }); - } - } - - return responseHeaders; + const useCorsProxy = typeof this.config.useCorsProxy !== "undefined" && this.config.useCorsProxy; + return performWebRequest(url, type, useCorsProxy, requestHeaders, expectedResponseHeaders); } }); From 5fb1fd1a80165cecfab2fa953e45abb1ea9da7f1 Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Sun, 6 Nov 2022 17:26:03 +0100 Subject: [PATCH 6/9] Removed check for corrdinates when checking cache data. The current implementation is faulty. I think it is useless as well, because mirrors are likely to only show weather for one location. If this is to be implemented, it should be done for both weather and stellar data. --- modules/default/weather/providers/yr.js | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/modules/default/weather/providers/yr.js b/modules/default/weather/providers/yr.js index d2b3e0ad44..a57dd17346 100644 --- a/modules/default/weather/providers/yr.js +++ b/modules/default/weather/providers/yr.js @@ -86,6 +86,7 @@ WeatherProvider.register("yr", { }, 5000); //Assume other fetch finished but failed to remove lock const attemptFetchWeather = setInterval(() => { if (!shouldWait) { + clearInterval(checkForGo); clearInterval(attemptFetchWeather); this.getWeatherDataFromYrOrCache(resolve, reject); } @@ -216,6 +217,7 @@ WeatherProvider.register("yr", { }, 5000); //Assume other fetch finished but failed to remove lock const attemptFetchWeather = setInterval(() => { if (!shouldWait) { + clearInterval(checkForGo); clearInterval(attemptFetchWeather); this.getStellarDataFromYrOrCache(resolve, reject); } @@ -232,11 +234,11 @@ WeatherProvider.register("yr", { let stellarData = this.getStellarDataFromCache(); const today = moment().format("YYYY-MM-DD"); const tomorrow = moment().add(1, "days").format("YYYY-MM-DD"); - if (stellarData && stellarData.today && stellarData.today.date === today && stellarData.tomorrow && stellarData.tomorrow.date === tomorrow && this.coordinatesAreCorrect(stellarData.today, stellarData.tomorrow)) { + if (stellarData && stellarData.today && stellarData.today.date === today && stellarData.tomorrow && stellarData.tomorrow.date === tomorrow) { Log.debug("Stellar data found in cache."); localStorage.removeItem("yrIsFetchingStellarData"); resolve(stellarData); - } else if (stellarData && stellarData.tomorrow && stellarData.tomorrow.date === today && this.coordinatesAreCorrect(stellarData.tomorrow)) { + } else if (stellarData && stellarData.tomorrow && stellarData.tomorrow.date === today) { Log.debug("stellar data for today found in cache, but not for tomorrow."); stellarData.today = stellarData.tomorrow; this.getStellarDataFromYr(tomorrow) @@ -284,17 +286,6 @@ WeatherProvider.register("yr", { } }, - coordinatesAreCorrect(todayData, tomorrowData) { - if (tomorrowData) - return ( - !todayData.location || - (Math.abs(parseFloat(this.config.lat) - parseFloat(todayData.location.latitude)) < 0.1 && - Math.abs(parseFloat(this.config.lon) - parseFloat(todayData.location.longitude)) < 0.1 && - (!tomorrowData.location || (Math.abs(parseFloat(this.config.lat) - parseFloat(tomorrowData.location.latitude)) < 0.1 && Math.abs(parseFloat(this.config.lon) - parseFloat(tomorrowData.location.longitude)) < 0.1))) - ); - return !todayData.location || (Math.abs(parseFloat(this.config.lat) - parseFloat(todayData.location.latitude)) < 0.1 && Math.abs(parseFloat(this.config.lon) - parseFloat(todayData.location.longitude)) < 0.1); - }, - getStellarDataFromCache() { const stellarData = localStorage.getItem("stellarData"); if (stellarData) { From 058b576735fc751a8726cb19c857555712a5156c Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Sun, 6 Nov 2022 21:27:17 +0100 Subject: [PATCH 7/9] Added unit tests for module utils. Because the fetch used requires a later node version, I only test the ones that work. The others gets a default test to avoid empty test suite. --- tests/unit/modules/default/utils_spec.js | 111 +++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/unit/modules/default/utils_spec.js diff --git a/tests/unit/modules/default/utils_spec.js b/tests/unit/modules/default/utils_spec.js new file mode 100644 index 0000000000..a384cd8e1f --- /dev/null +++ b/tests/unit/modules/default/utils_spec.js @@ -0,0 +1,111 @@ +const { performWebRequest } = require("../../../../modules/default/utils.js"); +const nodeVersion = process.version.match(/^v(\d+)\.*/)[1]; + +describe("Utils tests", () => { + describe("The performWebRequest-method", () => { + if (nodeVersion > 18) { + const locationHost = "localhost:8080"; + const locationProtocol = "http"; + + let fetchResponse; + let fetchMock; + let url; + + beforeEach(() => { + fetchResponse = new Response(); + global.fetch = jest.fn(() => Promise.resolve(fetchResponse)); + fetchMock = global.fetch; + + url = "www.test.com"; + }); + + describe("When using cors proxy", () => { + Object.defineProperty(global, "location", { + value: { + host: locationHost, + protocol: locationProtocol + } + }); + + test("Calls correct URL once", async () => { + const urlToCall = "http://www.test.com/path?param1=value1"; + url = urlToCall; + + await performWebRequest(url, "json", true); + + expect(fetchMock.mock.calls.length).toBe(1); + expect(fetchMock.mock.calls[0][0]).toBe(`${locationProtocol}//${locationHost}/cors?url=${urlToCall}`); + }); + + test("Sends correct headers", async () => { + const urlToCall = "http://www.test.com/path?param1=value1"; + url = urlToCall; + const headers = [ + { name: "header1", value: "value1" }, + { name: "header2", value: "value2" } + ]; + + await performWebRequest(url, "json", true, headers); + + expect(fetchMock.mock.calls.length).toBe(1); + expect(fetchMock.mock.calls[0][0]).toBe(`${locationProtocol}//${locationHost}/cors?sendheaders=header1:value1,header2:value2&url=${urlToCall}`); + }); + }); + + describe("When not using cors proxy", () => { + test("Calls correct URL once", async () => { + const urlToCall = "http://www.test.com/path?param1=value1"; + url = urlToCall; + + await performWebRequest(url); + + expect(fetchMock.mock.calls.length).toBe(1); + expect(fetchMock.mock.calls[0][0]).toBe(urlToCall); + }); + + test("Sends correct headers", async () => { + const urlToCall = "http://www.test.com/path?param1=value1"; + url = urlToCall; + const headers = [ + { name: "header1", value: "value1" }, + { name: "header2", value: "value2" } + ]; + + await performWebRequest(url, "json", false, headers); + + const expectedHeaders = { headers: { header1: "value1", header2: "value2" } }; + expect(fetchMock.mock.calls.length).toBe(1); + expect(fetchMock.mock.calls[0][1]).toStrictEqual(expectedHeaders); + }); + }); + + describe("When receiving json format", () => { + test("Returns undefined when no data is received", async () => { + const response = await performWebRequest(url); + + expect(response).toBe(undefined); + }); + + test("Returns object when data is received", async () => { + fetchResponse = new Response('{"body": "some content"}'); + + const response = await performWebRequest(url); + + expect(response.body).toBe("some content"); + }); + + test("Returns expected headers when data is received", async () => { + fetchResponse = new Response('{"body": "some content"}', { headers: { header1: "value1", header2: "value2" } }); + + const response = await performWebRequest(url, "json", false, undefined, ["header1"]); + + expect(response.headers.length).toBe(1); + expect(response.headers[0].name).toBe("header1"); + expect(response.headers[0].value).toBe("value1"); + }); + }); + } else { + test("Always ok, need one test", () => {}); + } + }); +}); From 24fc34a4b50f44cecf096a240bb7626aaa5046c3 Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Mon, 7 Nov 2022 18:48:53 +0100 Subject: [PATCH 8/9] Removed module.export, not necassary. --- modules/default/utils.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/default/utils.js b/modules/default/utils.js index 266f613f62..fb2cab8f31 100644 --- a/modules/default/utils.js +++ b/modules/default/utils.js @@ -141,6 +141,7 @@ const getHeadersFromResponse = (expectedResponseHeaders, response) => { return responseHeaders; }; -module.exports = { - performWebRequest -}; +if (typeof module !== "undefined") + module.exports = { + performWebRequest + }; From 4dc8469e2b495d5921d661b1012763420b3cd0cf Mon Sep 17 00:00:00 2001 From: Magnus Marthinsen Date: Mon, 7 Nov 2022 19:26:02 +0100 Subject: [PATCH 9/9] Avoid crash on faulty lon- and lat-configs. Avoid crashing, just trim the coordinates. I decided to trim instead of round for simplicity. --- modules/default/weather/providers/yr.js | 28 ++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/modules/default/weather/providers/yr.js b/modules/default/weather/providers/yr.js index a57dd17346..cc21611be3 100644 --- a/modules/default/weather/providers/yr.js +++ b/modules/default/weather/providers/yr.js @@ -177,17 +177,19 @@ WeatherProvider.register("yr", { throw new Error("Longitude not provided."); } - const lat = this.config.lat.toString(); - const lon = this.config.lon.toString(); + let lat = this.config.lat.toString(); + let lon = this.config.lon.toString(); const altitude = this.config.altitude ?? 0; if (lat.includes(".") && lat.split(".")[1].length > 4) { - Log.error("Latitude is too specific. Do not use more than four decimals."); - throw new Error("Latitude too specific."); + Log.warn("Latitude is too specific for weather data. Do not use more than four decimals. Trimming to maximum length."); + const latParts = lat.split("."); + lat = `${latParts[0]}.${latParts[1].substring(0, 4)}`; } if (lon.includes(".") && lon.split(".")[1].length > 4) { - Log.error("Longitude is too specific. Do not use more than four decimals."); - throw new Error("Longitude too specific."); + Log.warn("Longitude is too specific for weather data. Do not use more than four decimals. Trimming to maximum length."); + const lonParts = lon.split("."); + lon = `${lonParts[0]}.${lonParts[1].substring(0, 4)}`; } return `${this.config.apiBase}/locationforecast/2.0/complete?&altitude=${altitude}&lat=${lat}&lon=${lon}`; @@ -318,17 +320,19 @@ WeatherProvider.register("yr", { throw new Error("Longitude not provided."); } - const lat = this.config.lat.toString(); - const lon = this.config.lon.toString(); + let lat = this.config.lat.toString(); + let lon = this.config.lon.toString(); const altitude = this.config.altitude ?? 0; if (lat.includes(".") && lat.split(".")[1].length > 4) { - Log.error("Latitude is too specific. Do not use more than four decimals."); - throw new Error("Latitude too specific."); + Log.warn("Latitude is too specific for stellar data. Do not use more than four decimals. Trimming to maximum length."); + const latParts = lat.split("."); + lat = `${latParts[0]}.${latParts[1].substring(0, 4)}`; } if (lon.includes(".") && lon.split(".")[1].length > 4) { - Log.error("Longitude is too specific. Do not use more than four decimals."); - throw new Error("Longitude too specific."); + Log.warn("Longitude is too specific for stellar data. Do not use more than four decimals. Trimming to maximum length."); + const lonParts = lon.split("."); + lon = `${lonParts[0]}.${lonParts[1].substring(0, 4)}`; } let utcOffset = moment().utcOffset() / 60;