Merge pull request #5 from wneessen/dev

v1.3.2

Author: Winni Neessen

.gitignore

@@ -8,8 +8,8 @@ lerna-debug.log*
 .vscode/*
 log/*
 !log/.gitkeep
-conf/*
-!conf/websitebench.conf
+config/*
+!config/websitebench.conf
 
 
 # Diagnostic reports (https://nodejs.org/api/report.html)

Dockerfile

@@ -8,10 +8,10 @@ RUN         /usr/bin/groupadd -r websitebench && /usr/bin/useradd -r -g websiteb
 COPY        ["LICENSE", "README.md", "package.json", "package-lock.json", "/opt/websiteBench/"]
 COPY        ["dist", "/opt/websiteBench/dist"]
 COPY        ["log", "/opt/websiteBench/log"]
-COPY        ["conf", "/opt/websiteBench/conf"]
+COPY        ["config", "/opt/websiteBench/config"]
 RUN         chown -R websitebench:websitebench /opt/websiteBench
 WORKDIR     /opt/websiteBench
 USER        websitebench
 RUN         npm install
-VOLUME      ["/opt/websiteBench/conf", "/opt/websiteBench/log"]
+VOLUME      ["/opt/websiteBench/config", "/opt/websiteBench/log"]
 ENTRYPOINT  ["/usr/bin/node", "dist/websiteBench.js"]

README.md

@@ -3,10 +3,12 @@ websiteBench will measure website performance and propagate the results into an
 
 ## Requirements
 This service requires some NodeJS and some modules to work:
-- [NodeJS](https://nodejs.org/en/)
-- [Google Puppeteer](https://pptr.dev/)
 - [Arg](https://www.npmjs.com/package/arg)
 - [Node-Influx](https://node-influx.github.io/)
+- [NodeJS](https://nodejs.org/en/)
+- [node-libcurl](https://github.com/JCMais/node-libcurl/)
+- [Google Puppeteer](https://pptr.dev/)
+- [Q](https://github.com/kriskowal/q)
 - [tslog](https://tslog.js.org/)
 
 The modules should be automagically be installed by running: ```npm install```
@@ -51,7 +53,7 @@ To run the Docker image simply issue the following command:
   ```
 - Run the docker image
   ```sh
-  $ docker run --security-opt seccomp=wb-seccomp.json -v /var/db/websiteBench/config/:/opt/websiteBench/conf/ -v /var/db/websiteBench/log/:/opt/websiteBench/log/ website-bench:prod -c conf/yourconfig.conf
+  $ docker run --security-opt seccomp=wb-seccomp.json -v /var/db/websiteBench/config/:/opt/websiteBench/config/ -v /var/db/websiteBench/log/:/opt/websiteBench/log/ website-bench:prod -c config/yourconfig.conf
   ```
   (You can add additional CLI parameters if needed)
 
@@ -63,7 +65,8 @@ The config accepts the following options:
         {
             "siteName": "Your Site No. 1",
             "siteUrl": "https://example.com/fooo.html",
-            "checkInterval": 130
+            "checkInterval": 130,
+            "checkType": "curl"
         },
         {
             "siteName": "Your other great Site",
@@ -79,14 +82,19 @@ The config accepts the following options:
         "database": "websitebench",
         "ignoressl": false
     },
-    "allowCaching": false
+    "allowCaching": false,
+	  "logLevel": "info",
+    "maxConcurrentJobs": 5,
+    "instanceName": "DC1-CGN-I1"
 }
 ```
 - ```websiteList (Array<IWebEntry>)```: List of websites to collect performance data on
 - ```influxDb (IInfluxDbConfig)```: InfluxDB config settings
 - ```allowCaching (Boolean)```: If set, the browser will not open Incognito instances, but simple new pages
 - ```userAgent (String)```: Override the browser User-Agent
+- ```instanceName (String)```: Name of the instance running the checks (will be tagged in InfluxDB accordingly)
 - ```logResErrors (Boolean)```: If set, resource errors of the web requests will be logged (same as ```--log-resource-errors```)
+- ```ignoreSslErrors (Boolean)```: If set, Chrome/cURL will not error on invalid SSL certificates
 - ```maxConcurrentJobs (Number)```: Number of concurrently executed performance checks
 - ```logLevel (ILogLevel)```: Define the loglevel
 
@@ -95,6 +103,7 @@ The config accepts the following options:
   -  ```siteName (String)```: The name of the site you are monitoring (will be the tag in the InfluxDB)
   -  ```siteUrl (String)```: The URL to monitor
   -  ```checkInterval (Number)```: The interval to perform the checks on (in seconds). Minimum value is 60 seconds.
+  -  ```checkType (String)```: The type of performance check to run. Can be either "curl" or "browser" (Default: browser)
 - ```IInfluxDbConfig (Object)```: Consists of the following settings:
   -  ```hostname (String)```: Hostname or IP of the InfluxDB server
   -  ```database (String)```: InfluxDB database name to store the metrics in
@@ -109,11 +118,11 @@ The server provides the following CLI parameters to override defaults
 
 - ```-c, --config <filepath> ```: Path to config file
 - ```-s, --secrets <filepath> ```: Path to secrets config file
-- ```--ignore-ssl-errors```: Ignore HTTPS errors
-- ```--log-resource-errors```: If set, resource errors of the web requests will be logged
 - ```-d, --debug```: Enable DEBUG mode (more logging)
 - ```--no-headless```: If set, the browser will start in non-headless mode
 - ```--no-sandbox```: If set, the browser is started in no-sandbox mode (**DANGEROUS**: Only use if you are sure what you are doing)
+- ```--ignore-ssl-errors```: Ignore HTTPS errors
+- ```--log-resource-errors```: If set, resource errors of the web requests will be logged
 - ```--browserpath <path to browser executabel>```: Run Puppeteer with a different browser (Chrome/Firefox supported)
 - ```--browsertype <chrome|firefox>```: Run Puppeteer with a different browser type (Requires: --browserpath to be set)
 

conf/websitebench.conf config/websitebench.confconf/websitebench.conf renamed to config/websitebench.conf

@@ -3,7 +3,8 @@
         {
             "siteName": "Your Site No. 1",
             "siteUrl": "https://example.com/fooo.html",
-            "checkInterval": 130
+            "checkInterval": 130,
+            "checkType": "curl"
         },
         {
             "siteName": "Your other great Site",
@@ -18,5 +19,9 @@
         "port": 443,
         "database": "websitebench",
         "ignoressl": false
-    }
+    },
+    "allowCaching": false
+	"logLevel": "info",
+    "maxConcurrentJobs": 5,
+    "instanceName": "DC1-CGN-I1"
 }

dist/lib/websiteBenchBrowser.d.ts

@@ -9,7 +9,8 @@ export default class WebsiteBenchBrowser {
     private logObj;
     private numOfRetries;
     constructor(browserObj: Puppeteer.Browser, configObj: IWebsiteBenchConfig, logObj: Logger);
-    processPage(websiteEntry: IWebsiteEntry): Promise<IPerformanceData>;
+    processPageWithBrowser(websiteEntry: IWebsiteEntry): Promise<IPerformanceData>;
+    processPageWithCurl(websiteEntry: IWebsiteEntry): Promise<IPerformanceData>;
     private eventTriggered;
     private errorTriggered;
     private processPerformanceData;

dist/lib/websiteBenchBrowser.d.ts.map

@@ -1,9 +1,30 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+const node_libcurl_1 = require("node-libcurl");
 const websiteBenchTools_1 = __importDefault(require("./websiteBenchTools"));
+const qObj = __importStar(require("q"));
 class WebsiteBenchBrowser {
     constructor(browserObj, configObj, logObj) {
         this.toolsObj = new websiteBenchTools_1.default();
@@ -13,7 +34,7 @@ class WebsiteBenchBrowser {
         this.configObj = configObj;
         this.logObj = logObj;
     }
-    async processPage(websiteEntry) {
+    async processPageWithBrowser(websiteEntry) {
         const webUrl = websiteEntry.siteUrl;
         const reqTimeout = websiteEntry.checkInterval - 1;
         let perfData = null;
@@ -87,6 +108,85 @@ class WebsiteBenchBrowser {
         };
         return perfData;
     }
+    async processPageWithCurl(websiteEntry) {
+        return new Promise((retFunc, rejFunc) => {
+            const webUrl = websiteEntry.siteUrl;
+            const reqTimeout = websiteEntry.checkInterval - 1;
+            const promiseArray = [];
+            let userAgent;
+            let perfData = null;
+            const perfDataTotal = {
+                totalDurTime: 0,
+                connectTime: 0,
+                dnsTime: 0,
+                ttfbTime: 0,
+                preTransfer: 0,
+                tlsHandshake: 0,
+                statusCode: 0,
+                statusCodes: []
+            };
+            if (this.configObj.userAgent) {
+                userAgent = this.configObj.userAgent;
+            }
+            else {
+                let browserUserAgent = node_libcurl_1.Curl.defaultUserAgent;
+                userAgent = `${browserUserAgent} websiteBench/${this.configObj.versionNum}`;
+            }
+            for (let runCount = 0; runCount < this.numOfRetries; runCount++) {
+                this.logObj.debug(`Starting performance data collection for ${webUrl} (Run: ${runCount})...`);
+                const deferObj = qObj.defer();
+                const curlObj = new node_libcurl_1.Curl();
+                curlObj.setOpt('URL', webUrl);
+                curlObj.setOpt('TIMEOUT', reqTimeout);
+                curlObj.setOpt('USERAGENT', userAgent);
+                curlObj.setOpt('DNS_SHUFFLE_ADDRESSES', true);
+                curlObj.setOpt('SSL_VERIFYHOST', this.configObj.ignoreSslErrors === true ? false : true);
+                curlObj.on('end', (statusCode, resData, resHeader, curlInstance) => {
+                    const tempPerf = {
+                        runNumber: runCount,
+                        totalDurTime: curlInstance.getInfo('TOTAL_TIME_T') / 1000,
+                        dnsTime: curlInstance.getInfo('NAMELOOKUP_TIME_T') / 1000,
+                        tlsHandshake: curlInstance.getInfo('APPCONNECT_TIME_T') / 1000,
+                        ttfbTime: curlInstance.getInfo('STARTTRANSFER_TIME_T') / 1000,
+                        preTransfer: curlInstance.getInfo('PRETRANSFER_TIME_T') / 1000,
+                        connectTime: curlInstance.getInfo('CONNECT_TIME_T') / 1000,
+                        statusCode: statusCode
+                    };
+                    deferObj.resolve(tempPerf);
+                    curlInstance.close();
+                });
+                curlObj.on('error', (errorObj) => {
+                    this.logObj.error(`Unable to fetch page via cURL: ${errorObj.message}`);
+                    this.logObj.debug(`Completed performance data collection with error for ${webUrl} (Run: ${runCount})...`);
+                });
+                curlObj.perform();
+                promiseArray.push(deferObj.promise);
+            }
+            qObj.all(promiseArray).then(resPromise => {
+                resPromise.forEach(curlPromise => {
+                    perfDataTotal.totalDurTime += curlPromise.totalDurTime;
+                    perfDataTotal.connectTime += curlPromise.connectTime;
+                    perfDataTotal.dnsTime += curlPromise.dnsTime;
+                    perfDataTotal.ttfbTime += curlPromise.ttfbTime;
+                    perfDataTotal.tlsHandshake += curlPromise.tlsHandshake;
+                    perfDataTotal.preTransfer += curlPromise.preTransfer;
+                    perfDataTotal.statusCodes.push(curlPromise.statusCode);
+                    this.logObj.debug(`Completed performance data collection for ${webUrl} (Run: ${curlPromise.runNumber})...`);
+                });
+            }).finally(() => {
+                perfData = {
+                    totalDurTime: (perfDataTotal.totalDurTime / this.numOfRetries),
+                    connectTime: (perfDataTotal.connectTime / this.numOfRetries),
+                    dnsTime: (perfDataTotal.dnsTime / this.numOfRetries),
+                    ttfbTime: (perfDataTotal.ttfbTime / this.numOfRetries),
+                    tlsHandshake: (perfDataTotal.tlsHandshake / this.numOfRetries),
+                    preTransfer: (perfDataTotal.preTransfer / this.numOfRetries),
+                };
+                perfData.statusCodesString = perfDataTotal.statusCodes.join(':');
+                retFunc(perfData);
+            });
+        });
+    }
     async eventTriggered(eventObj) {
         if (this.toolsObj.eventIsDialog(eventObj)) {
             eventObj.dismiss();

dist/lib/websiteBenchBrowser.js

@@ -4,6 +4,7 @@ export default class WebsiteBenchConfig {
     private _configObj;
     private _versionNum;
     private _allowCaching;
+    private _ignoreSslErrors;
     private _logResourceErrors;
     private _maxConcurrentJobs;
     private _minCheckInterval;

dist/lib/websiteBenchConfig.d.ts

@@ -5,8 +5,9 @@ const process_1 = require("process");
 class WebsiteBenchConfig {
     constructor(confFiles, logObj) {
         this._configObj = {};
-        this._versionNum = '1.2.3';
+        this._versionNum = '1.3.2';
         this._allowCaching = false;
+        this._ignoreSslErrors = false;
         this._logResourceErrors = false;
         this._maxConcurrentJobs = 5;
         this._minCheckInterval = 30;
@@ -18,7 +19,8 @@ class WebsiteBenchConfig {
             allowCaching: this._allowCaching,
             maxConcurrentJobs: this._maxConcurrentJobs,
             logResErrors: this._logResourceErrors,
-            versionNum: this._versionNum,
+            ignoreSslErrors: this._ignoreSslErrors,
+            versionNum: this._versionNum
         }, confFileData);
         this._configObj.influxDb = { ...this._configObj.influxDb, ...secretFileData.influxDb };
         try {
@@ -46,7 +48,7 @@ class WebsiteBenchConfig {
         return this._configObj;
     }
     checkMandatory() {
-        const mandatoryProps = ['maxConcurrentJobs', 'allowCaching', 'websiteList', 'influxDb'];
+        const mandatoryProps = ['maxConcurrentJobs', 'allowCaching', 'websiteList', 'influxDb', 'instanceName'];
         const mandatoryInflux = ['hostname', 'database', 'username', 'password'];
         let missingProps = null;
         for (const objProp of mandatoryProps) {
@@ -87,6 +89,11 @@ class WebsiteBenchConfig {
                 configError.errorProperty = 'websiteList => siteUrl';
                 configError.errorMessage = 'Not all website list entries have a "siteUrl" property or the value is not a string';
             }
+            if ('checkType' in websiteEntry && typeof websiteEntry.checkType !== 'string') {
+                configError.hasError = true;
+                configError.errorProperty = 'websiteList => checkType';
+                configError.errorMessage = 'Value of "checkType" is not a string';
+            }
             if (!('checkInterval' in websiteEntry) || typeof websiteEntry.checkInterval !== 'number') {
                 configError.hasError = true;
                 configError.errorProperty = 'websiteList => checkInterval';
@@ -107,6 +114,11 @@ class WebsiteBenchConfig {
                 configError.errorProperty = 'siteUrl';
                 configError.errorMessage = `"siteUrl" of website entry "${websiteEntry.siteName}" cannot be empty`;
             }
+            if ('checkType' in websiteEntry && (websiteEntry.checkType.toLowerCase() !== 'browser' && websiteEntry.checkType.toLowerCase() !== 'curl')) {
+                configError.hasError = true;
+                configError.errorProperty = 'checkType';
+                configError.errorMessage = `"checkType" of website entry "${websiteEntry.siteName}" has to be either "browser" or "curl"`;
+            }
             try {
                 new URL(websiteEntry.siteUrl);
             }