Merge pull request #592 from buster95/feature/docker

[Docker] Phase 1: Splitting scraper from API

Author: ebwinters

.github/workflows/lint.yml

@@ -1,14 +1,13 @@
 name: Eslint
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
 jobs:
   eslint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - run: npm install
       - run: npm run lint
-  

.github/workflows/test.yml

@@ -5,7 +5,7 @@ on:
   pull_request:
     branches: [master]
   schedule:
-    - cron: '0 */2 * * *'
+    - cron: "0 */2 * * *"
 jobs:
   unittests:
     runs-on: ubuntu-latest
@@ -15,6 +15,9 @@ jobs:
     steps:
       - name: Git checkout
         uses: actions/checkout@v2
+      - uses: zcong1993/setup-timezone@master
+        with:
+          timezone: America/Denver
       - name: Start redis server
         uses: supercharge/redis-github-action@1.1.0
         with:

.gitignore

@@ -2,6 +2,7 @@ node_modules/
 json.sqlite
 .vscode/
 config.json
+docker.env
 package-lock.json
 dump.rdb
 yarn.lock

.mocharc.js

@@ -0,0 +1,3 @@
+module.exports = {
+    file: ['./tests/mochaSetup.spec.js']
+};

config.example.json config/config.example.jsonconfig.example.json renamed to config/config.example.json

@@ -0,0 +1,15 @@
+const keys = require('./config.keys.json');
+let config;
+
+try {
+	config = require('./config.json');
+} catch (err) {
+	config = require('./config.example.json');
+}
+
+console.log(process.env.covid_api_mode);
+
+module.exports = {
+	config,
+	keys
+};

config.keys.json config/config.keys.jsonconfig.keys.json renamed to config/config.keys.json

@@ -17,11 +17,14 @@
   ],
   "scripts": {
     "start": "node server.js",
+    "start:scraper": "node server_scraper.js",
     "start:dev": "nodemon --watch routes --watch scrapers --watch utils ./server.js ",
     "lint": "eslint '**/*.js' --ignore-pattern node_modules/",
     "lint:fix": "eslint ./**/*.js --fix --ignore-pattern node_modules/",
     "lint:win32": "eslint ./**/*.js --ignore-pattern node_modules/",
-    "test": "mocha ./tests --exit  --timeout 50000"
+    "test": "mocha ./tests --exit  --timeout 60000",
+    "docker-start": "docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d --build",
+    "docker-test": "docker-compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit --build"
   },
   "dependencies": {
     "@sentry/node": "5.12.2",
@@ -45,4 +48,4 @@
     "mocha": "^7.1.1",
     "nodemon": "^2.0.2"
   }
-}
+}

config/index.js

@@ -2,21 +2,15 @@
 const Redis = require('ioredis');
 
 // LOCAL FUNCTIONS
+const logger = require('../utils/logger');
 const getWorldometerPage = require('../scrapers/getWorldometers');
 const getStates = require('../scrapers/getStates');
 const jhuLocations = require('../scrapers/jhuLocations');
 const historical = require('../scrapers/historical');
 const nytData = require('../scrapers/nytData');
 
 // KEYS
-const keys = require('../config.keys.json');
-
-let config;
-try {
-	config = require('../config.json');
-} catch (err) {
-	config = require('../config.example.json');
-}
+const { config, keys } = require('../config');
 
 const redis = new Redis(config.redis.host, {
 	password: config.redis.password,
@@ -32,6 +26,20 @@ module.exports = {
 		getStates,
 		jhuLocations,
 		historical,
-		nytData
+		nytData,
+		executeScraper: async () => {
+			await Promise.all([
+				getWorldometerPage(keys, redis),
+				getStates(keys, redis),
+				jhuLocations.jhudataV2(keys, redis),
+				historical.historicalV2(keys, redis),
+				historical.getHistoricalUSADataV2(keys, redis)
+			]);
+			logger.info('Finished scraping!');
+		},
+		executeScraperNYTData: async () => {
+			await nytData(keys, redis);
+			logger.info('Finished NYT scraping!');
+		}
 	}
 };

package.json

@@ -75,6 +75,7 @@ const getStates = async (keys, redis) => {
 		response = await axios.get('https://www.worldometers.info/coronavirus/country/us/');
 	} catch (err) {
 		logger.err('Error: Requesting GetStates failed!', err);
+		return;
 	}
 	const html = cheerio.load(response.data);