mock s3 v2

.gitignore

@@ -7,4 +7,5 @@ node-webkit.app
 npm-debug.log
 .nyc_output
 coverage
-test/*/package-lock.json
+test/*/package-lock.json
+test/app7/build-tmp-napi-*

.travis.yml

@@ -9,6 +9,8 @@ services:
 # have more than one run with the same node version
 # otherwise they will clobber each other
 #####
+env:
+  node_pre_gyp_mock_s3=true
 
 matrix:
   include:
@@ -17,6 +19,7 @@ matrix:
       node_js: 10
       before_install:
         - npm install request
+        - npm install --package-lock-only
       script:
         - npm run lint
         - npm run coverage

appveyor.yml

@@ -1,6 +1,9 @@
 os: Visual Studio 2015
 
 environment:
+  global:
+    node_pre_gyp_mock_s3: true
+
   matrix:
     - nodejs_version: 10
     - nodejs_version: 12

lib/info.js

@@ -10,13 +10,13 @@ const s3_setup = require('./util/s3_setup.js');
 const config = require('rc')('node_pre_gyp', { acl: 'public-read' });
 
 function info(gyp, argv, callback) {
-  const AWS = require('aws-sdk');
   const package_json = gyp.package_json;
   const opts = versioning.evaluate(package_json, gyp.opts);
+
   s3_setup.detect(opts.hosted_path, config);
-  AWS.config.update(config);
-  const s3 =  new AWS.S3();
-  const s3_opts = {  Bucket: config.bucket,
+  const s3 = s3_setup.get_s3(config);
+  const s3_opts = {
+    Bucket: config.bucket,
     Prefix: config.prefix
   };
   s3.listObjects(s3_opts, (err, meta) => {

lib/node-pre-gyp.js

@@ -9,6 +9,13 @@ module.exports = exports;
  * Module dependencies.
  */
 
+// load mocking control function for accessing s3 via https. the function is a noop always returning
+// false if not mocking.
+exports.mockS3Http = require('./util/s3_setup').get_mockS3Http();
+exports.mockS3Http('on');
+const mocking = exports.mockS3Http('get');
+
+
 const fs = require('fs');
 const path = require('path');
 const nopt = require('nopt');
@@ -38,6 +45,10 @@ const aliases = {};
 // differentiate node-pre-gyp's logs from npm's
 log.heading = 'node-pre-gyp';
 
+if (mocking) {
+  log.warn(`mocking s3 to ${process.env.node_pre_gyp_mock_s3}`);
+}
+
 exports.find = require('./pre-binding').find;
 
 //
@@ -287,4 +298,3 @@ Object.defineProperty(proto, 'version', {
   },
   enumerable: true
 });
-

lib/publish.js

@@ -15,7 +15,6 @@ const url = require('url');
 const config = require('rc')('node_pre_gyp', { acl: 'public-read' });
 
 function publish(gyp, argv, callback) {
-  const AWS = require('aws-sdk');
   const package_json = gyp.package_json;
   const napi_build_version = napi.get_napi_build_version_from_command_args(argv);
   const opts = versioning.evaluate(package_json, gyp.opts, napi_build_version);
@@ -24,16 +23,19 @@ function publish(gyp, argv, callback) {
     if (!found) {
       return callback(new Error('Cannot publish because ' + tarball + ' missing: run `node-pre-gyp package` first'));
     }
+
     log.info('publish', 'Detecting s3 credentials');
     s3_setup.detect(opts.hosted_path, config);
+
+    const s3 = s3_setup.get_s3(config);
+    const s3_opts = {
+      Bucket: config.bucket,
+      Prefix: config.prefix
+    };
     const key_name = url.resolve(config.prefix, opts.package_name);
     log.info('publish', 'Authenticating with s3');
-    AWS.config.update(config);
-    log.info('publish', AWS.config);
-    const s3 =  new AWS.S3();
-    const s3_opts = {  Bucket: config.bucket,
-      Key: key_name
-    };
+    log.info('publish', config);
+
     const remote_package = 'https://' + s3_opts.Bucket + '.s3.amazonaws.com/' + s3_opts.Key;
     log.info('publish', 'Checking for existing binary at ' + remote_package);
     s3.headObject(s3_opts, (err, meta) => {
@@ -42,15 +44,17 @@ function publish(gyp, argv, callback) {
         // we are safe to publish because
         // the object does not already exist
         log.info('publish', 'Preparing to put object');
-        const s3_put = new AWS.S3();
-        const s3_put_opts = {  ACL: config.acl,
+
+        // const s3_put = new AWS.S3();
+        const s3_put_opts = {
+          ACL: config.acl,
           Body: fs.createReadStream(tarball),
           Bucket: config.bucket,
           Key: key_name
         };
         log.info('publish', 'Putting object', s3_put_opts.ACL, s3_put_opts.Bucket, s3_put_opts.Key);
         try {
-          s3_put.putObject(s3_put_opts, (err2, resp) => {
+          s3.putObject(s3_put_opts, (err2, resp) => {
             log.info('publish', 'returned from putting object');
             if (err2) {
               log.info('publish', 's3 putObject error: "' + err2 + '"');

lib/unpublish.js

@@ -12,15 +12,15 @@ const url = require('url');
 const config = require('rc')('node_pre_gyp', { acl: 'public-read' });
 
 function unpublish(gyp, argv, callback) {
-  const AWS = require('aws-sdk');
   const package_json = gyp.package_json;
   const napi_build_version = napi.get_napi_build_version_from_command_args(argv);
   const opts = versioning.evaluate(package_json, gyp.opts, napi_build_version);
+
   s3_setup.detect(opts.hosted_path, config);
-  AWS.config.update(config);
+  const s3 = s3_setup.get_s3(config);
   const key_name = url.resolve(config.prefix, opts.package_name);
-  const s3 =  new AWS.S3();
-  const s3_opts = {  Bucket: config.bucket,
+  const s3_opts = {
+    Bucket: config.bucket,
     Key: key_name
   };
   s3.headObject(s3_opts, (err, meta) => {

lib/util/s3_setup.js

@@ -3,6 +3,8 @@
 module.exports = exports;
 
 const url = require('url');
+const fs = require('fs');
+const path = require('path');
 
 module.exports.detect = function(to, config) {
   const uri = url.parse(to);
@@ -24,3 +26,130 @@ module.exports.detect = function(to, config) {
     }
   }
 };
+
+module.exports.get_s3 = function(config) {
+
+  if (process.env.node_pre_gyp_mock_s3) {
+    // here we're mocking. node_pre_gyp_mock_s3 is the scratch directory
+    // for the mock code.
+    const AWSMock = require('mock-aws-s3');
+    const os = require('os');
+
+    AWSMock.config.basePath = `${os.tmpdir()}/mock`;
+
+    const s3 = AWSMock.S3();
+
+    // wrapped callback maker. fs calls return code of ENOENT but AWS.S3 returns
+    // NotFound.
+    const wcb = (fn) => (err, ...args) => {
+      if (err && err.code === 'ENOENT') {
+        err.code = 'NotFound';
+      }
+      return fn(err, ...args);
+    };
+
+    return {
+      listObjects(params, callback) {
+        return s3.listObjects(params, wcb(callback));
+      },
+      headObject(params, callback) {
+        return s3.headObject(params, wcb(callback));
+      },
+      deleteObject(params, callback) {
+        return s3.deleteObject(params, wcb(callback));
+      },
+      putObject(params, callback) {
+        return s3.putObject(params, wcb(callback));
+      }
+    };
+  }
+
+  // if not mocking then setup real s3.
+  const AWS = require('aws-sdk');
+
+  AWS.config.update(config);
+  const s3 = new AWS.S3();
+
+  // need to change if additional options need to be specified.
+  return {
+    listObjects(params, callback) {
+      return s3.listObjects(params, callback);
+    },
+    headObject(params, callback) {
+      return s3.headObject(params, callback);
+    },
+    deleteObject(params, callback) {
+      return s3.deleteObject(params, callback);
+    },
+    putObject(params, callback) {
+      return s3.putObject(params, callback);
+    }
+  };
+
+
+
+};
+
+//
+// function to get the mocking control function. if not mocking it returns a no-op.
+//
+// if mocking it sets up the mock http interceptors that use the mocked s3 file system
+// to fulfill reponses.
+module.exports.get_mockS3Http = function() {
+  let mock_s3 = false;
+  if (!process.env.node_pre_gyp_mock_s3) {
+    return () => mock_s3;
+  }
+
+  const nock = require('nock');
+  // the bucket used for testing, as addressed by https.
+  const host = 'https://mapbox-node-pre-gyp-public-testing-bucket.s3.us-east-1.amazonaws.com';
+  const mockDir = process.env.node_pre_gyp_mock_s3 + '/mapbox-node-pre-gyp-public-testing-bucket';
+
+  // function to setup interceptors. they are "turned off" by setting mock_s3 to false.
+  const mock_http = () => {
+    // eslint-disable-next-line no-unused-vars
+    function get(uri, requestBody) {
+      const filepath = path.join(mockDir, uri.replace('%2B', '+'));
+
+      try {
+        fs.accessSync(filepath, fs.constants.R_OK);
+      } catch (e) {
+        return [404, 'not found\n'];
+      }
+
+      // the mock s3 functions just write to disk, so just read from it.
+      return [200, fs.createReadStream(filepath)];
+    }
+
+    // eslint-disable-next-line no-unused-vars
+    return nock(host)
+      .persist()
+      .get(() => mock_s3) // mock any uri for s3 when true
+      .reply(get);
+  };
+
+  // setup interceptors. they check the mock_s3 flag to determine whether to intercept.
+  mock_http(nock, host, mockDir);
+  // function to turn matching all requests to s3 on/off.
+  const mockS3Http = (action) => {
+    const previous = mock_s3;
+    if (action === 'off') {
+      mock_s3 = false;
+    } else if (action === 'on') {
+      mock_s3 = true;
+    } else if (action !== 'get') {
+      throw new Error(`illegal action for setMockHttp ${action}`);
+    }
+    return previous;
+  };
+
+  // call mockS3Http with the argument
+  // - 'on' - turn it on
+  // - 'off' - turn it off (used by fetch.test.js so it doesn't interfere with redirects)
+  // - 'get' - return true or false for 'on' or 'off'
+  return mockS3Http;
+};
+
+
+

package.json

@@ -38,6 +38,7 @@
     "codecov": "^3.8.1",
     "eslint": "^7.16.0",
     "eslint-plugin-node": "^11.1.0",
+    "mock-aws-s3": "^4.0.1",
     "nock": "^12.0.3",
     "node-addon-api": "^3.1.0",
     "nyc": "^15.1.0",