Make http 429 and 206 codes return alive

README.md

@@ -48,6 +48,9 @@ Parameters:
   * `ignorePatterns` an array of objects holding regular expressions which a link is checked against and skipped for checking in case of a match. Example: `[{ pattern: /foo/ }]`
   * `replacementPatterns` an array of objects holding regular expressions which are replaced in a link with their corresponding replacement string. This behavior allows (for example) to adapt to certain platform conventions hosting the Markdown. Example: `[{ pattern: /^.attachments/, replacement: "file://some/conventional/folder/.attachments" }]`
   * `ignoreDisable` if this is `true` then disable comments are ignored.
+  * `retryOn429` if this is `true` then retry request when response is an HTTP code 429 after the duration indicated by `retry-after` header.
+  * `aliveStatusCodes` a list of HTTP codes to consider as alive.
+    Example: `[200,206]`
 * `callback` function which accepts `(err, results)`.
   * `err` an Error object when the operation cannot be completed, otherwise `null`.
   * `results` an array of objects with the following properties:
@@ -129,17 +132,16 @@ If not supplied, the tool reads from standard input.
 #### Usage
 
 ```
-
-  Usage: markdown-link-check [options] [filenameOrUrl]
-
-  Options:
-
-    -p, --progress         show progress bar
-    -c, --config [config]  apply a configuration file (JSON)
-    -q, --quiet            display errors only
-    -v, --verbose          displays detailed error information 
-    -h, --help             output usage information
-
+Usage: markdown-link-check [options] [filenameOrUrl]
+
+Options:
+  -p, --progress         show progress bar
+  -c, --config [config]  apply a config file (JSON), holding e.g. url specific header configuration
+  -q, --quiet            displays errors only
+  -v, --verbose          displays detailed error information
+  -a, --alive <code>     comma separated list of HTTP code to be considered as alive
+  -r, --retry            retry after the duration indicated in 'retry-after' header when HTTP code is 429
+  -h, --help             display help for command
 ```
 
 ##### Config file format
@@ -149,6 +151,8 @@ If not supplied, the tool reads from standard input.
 * `ignorePatterns`: An array of objects holding regular expressions which a link is checked against and skipped for checking in case of a match.
 * `replacementPatterns`: An array of objects holding regular expressions which are replaced in a link with their corresponding replacement string. This behavior allows (for example) to adapt to certain platform conventions hosting the Markdown.
 * `httpHeaders`: The headers are only applied to links where the link **starts with** one of the supplied URLs in the `urls` section.
+* `retryOn429` if this is `true` then retry request when response is an HTTP code 429 after the duration indicated by `retry-after` header.
+* `aliveStatusCodes` a list of HTTP codes to consider as alive.
 
 **Example:**
 
@@ -175,7 +179,9 @@ If not supplied, the tool reads from standard input.
 				"Foo": "Bar"
 			}
 		}
-	]
+	],
+  "retryOn429":true,
+  "aliveStatusCodes":[200, 206]
 }
 ```
 

index.js

@@ -72,6 +72,7 @@ module.exports = function markdownLinkCheck(markdown, opts, callback) {
         }
 
         linkCheck(link, opts, function (err, result) {
+
             if (opts.showProgressBar) {
                 bar.tick();
             }

markdown-link-check

@@ -21,11 +21,19 @@ const opts = {};
 let filenameForOutput = '';
 let stream = process.stdin; // read from stdin unless a filename is given
 
+function commaSeparatedCodesList(value, dummyPrevious) {
+    return value.split(',').map(function(item) {
+      return parseInt(item, 10);
+    });
+}
+
 program
     .option('-p, --progress', 'show progress bar')
     .option('-c, --config [config]', 'apply a config file (JSON), holding e.g. url specific header configuration')
     .option('-q, --quiet', 'displays errors only')
     .option('-v, --verbose', 'displays detailed error information')
+    .option('-a, --alive <code>', 'comma separated list of HTTP codes to be considered as alive', commaSeparatedCodesList)
+    .option('-r, --retry', 'retry after the duration indicated in \'retry-after\' header when HTTP code is 429')
     .arguments('[filenameOrUrl]')
     .action(function (filenameOrUrl) {
     filenameForOutput = filenameOrUrl;
@@ -69,6 +77,8 @@ program
 opts.showProgressBar = (program.progress === true); // force true or undefined to be true or false.
 opts.quiet = (program.quiet === true);
 opts.verbose = (program.verbose === true);
+opts.retryOn429 = (program.retry === true);
+opts.aliveStatusCodes = program.alive;
 
 let markdown = ''; // collect the markdown data, then process it
 
@@ -104,6 +114,8 @@ stream
                     opts.replacementPatterns = config.replacementPatterns;
                     opts.httpHeaders = config.httpHeaders;
                     opts.ignoreDisable = config.ignoreDisable;
+                    opts.retryOn429 = config.retryOn429;
+                    opts.aliveStatusCodes = config.aliveStatusCodes;
 
                     runMarkdownLinkCheck(markdown, opts);
                 });

test/markdown-link-check.test.js

@@ -9,18 +9,38 @@ const markdownLinkCheck = require('../');
 
 describe('markdown-link-check', function () {
 
+    // add a longer timeout on tests so we can really test real cases.
+    // Mocha default is 2s, make it 5s here.
+    this.timeout(5000);
+
     let baseUrl;
 
     before(function (done) {
         const app = express();
 
+        var laterRetryCount = 0;
+
         app.head('/nohead', function (req, res) {
             res.sendStatus(405); // method not allowed
         });
         app.get('/nohead', function (req, res) {
             res.sendStatus(200);
         });
 
+        app.get('/partial', function (req, res) {
+            res.sendStatus(206);
+        });
+
+        app.get('/later', function (req, res) {
+            if(laterRetryCount<2){
+              laterRetryCount++;
+              res.append('retry-after', '2s');
+              res.sendStatus(429);
+            }else{
+              res.sendStatus(200);
+            }
+        });
+
         app.get('/foo/redirect', function (req, res) {
             res.redirect('/foo/bar');
         });
@@ -75,7 +95,9 @@ describe('markdown-link-check', function () {
                         urls: [baseUrl + '/basic-auth'],
                         headers: { 'Authorization': 'Basic Zm9vOmJhcg==', 'Foo': 'Bar' }
                     }
-                ] 
+                ],
+                "aliveStatusCodes":[200, 206],
+                "retryOn429":true
             }, function (err, results) {
             expect(err).to.be(null);
             expect(results).to.be.an('array');
@@ -108,6 +130,12 @@ describe('markdown-link-check', function () {
                 // replaced
                 { statusCode: 200, status: 'alive' },
 
+                // request rate limit return 429, retry later and get 200
+                { statusCode: 200, status: 'alive' },
+
+                // partial
+                { statusCode: 206, status: 'alive' },
+
                 // hello image
                 { statusCode: 200, status: 'alive' },
 

test/sample.md

@@ -11,6 +11,8 @@ This is a test file:
 * [basic-auth](%%BASE_URL%%/basic-auth) (alive)
 * [ignored](%%BASE_URL%%/something/not-working-and-ignored/something) (ignored)
 * [replaced](%%BASE_URL%%/boo/bar)
+* [later](%%BASE_URL%%/later)
+* [partial](%%BASE_URL%%/partial)
 
 ![img](%%BASE_URL%%/hello.jpg) (alive)
 ![img](hello.jpg) (alive)