From 2fc17b4ef7be8471eb698d436d2f1102a63c96b1 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Tue, 20 May 2014 21:40:33 -0700 Subject: [PATCH 01/23] License and contribution agreement --- CONTRIBUTING.md | 187 ++++++++++++++++++++++++++++++++++++++++++++++++ LICENSE.md | 7 ++ 2 files changed, 194 insertions(+) create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2654cd3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,187 @@ +### Contributing ### + +Thank you for your interest in strong-build, an open source project +administered by StrongLoop. + +Contributing to strong-build is easy. In a few simple steps: + + * Ensure that your effort is aligned with the project's roadmap by + talking to the maintainers, especially if you are going to spend a + lot of time on it. + + * Make something better or fix a bug. + + * Adhere to code style outlined in the [Google C++ Style Guide][] and + [Google Javascript Style Guide][]. + + * [Sign your patches](#signing-patches) to indicate that your are + making your contribution available under the terms of the + [Contributor License Agreement](#contributor-license-agreement). + + * Submit a pull request through Github. + + +### Signing patches ### + +Like many open source projects, we need a contributor license agreement +from you before we can merge in your changes. + +In summary, by submitting your code, you are granting us a right to use +that code under the terms of this Agreement, including providing it to +others. You are also certifying that you wrote it, and that you are +allowed to license it to us. You are not giving up your copyright in +your work. The license does not change your rights to use your own +contributions for any other purpose. + +Contributor License Agreements are important because they define the +chain of ownership of a piece of software. Some companies won't allow +the use of free software without clear agreements around code ownership. +That's why many open source projects collect similar agreements from +contributors. The CLA here is based on the Apache CLA. + +To signify your agreement to these terms, add the following line to the +bottom of your commit message. Use your real name and an actual e-mail +address. + +``` +Signed-off-by: Random J Developer +``` + +Alternatively you can use the git command line to automatically add this +line, as follows: + +``` +$ git commit -sm "Replace rainbows by unicorns" +``` + + +### Contributor License Agreement ### + +``` + Individual Contributor License Agreement + + By signing this Individual Contributor License Agreement + ("Agreement"), and making a Contribution (as defined below) to + StrongLoop, Inc. ("StrongLoop"), You (as defined below) accept and + agree to the following terms and conditions for Your present and + future Contributions submitted to StrongLoop. Except for the license + granted in this Agreement to StrongLoop and recipients of software + distributed by StrongLoop, You reserve all right, title, and interest + in and to Your Contributions. + + 1. Definitions + + "You" or "Your" shall mean the copyright owner or the individual + authorized by the copyright owner that is entering into this + Agreement with StrongLoop. + + "Contribution" shall mean any original work of authorship, + including any modifications or additions to an existing work, that + is intentionally submitted by You to StrongLoop for inclusion in, + or documentation of, any of the products owned or managed by + StrongLoop ("Work"). For purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication + sent to StrongLoop or its representatives, including but not + limited to communication or electronic mailing lists, source code + control systems, and issue tracking systems that are managed by, + or on behalf of, StrongLoop for the purpose of discussing and + improving the Work, but excluding communication that is + conspicuously marked or otherwise designated in writing by You as + "Not a Contribution." + + 2. You Grant a Copyright License to StrongLoop + + Subject to the terms and conditions of this Agreement, You hereby + grant to StrongLoop and recipients of software distributed by + StrongLoop, a perpetual, worldwide, non-exclusive, no-charge, + royalty-free, irrevocable copyright license to reproduce, prepare + derivative works of, publicly display, publicly perform, + sublicense, and distribute Your Contributions and such derivative + works under any license and without any restrictions. + + 3. You Grant a Patent License to StrongLoop + + Subject to the terms and conditions of this Agreement, You hereby + grant to StrongLoop and to recipients of software distributed by + StrongLoop a perpetual, worldwide, non-exclusive, no-charge, + royalty-free, irrevocable (except as stated in this Section) + patent license to make, have made, use, offer to sell, sell, + import, and otherwise transfer the Work under any license and + without any restrictions. The patent license You grant to + StrongLoop under this Section applies only to those patent claims + licensable by You that are necessarily infringed by Your + Contributions(s) alone or by combination of Your Contributions(s) + with the Work to which such Contribution(s) was submitted. If any + entity institutes a patent litigation against You or any other + entity (including a cross-claim or counterclaim in a lawsuit) + alleging that Your Contribution, or the Work to which You have + contributed, constitutes direct or contributory patent + infringement, any patent licenses granted to that entity under + this Agreement for that Contribution or Work shall terminate as + of the date such litigation is filed. + + 4. You Have the Right to Grant Licenses to StrongLoop + + You represent that You are legally entitled to grant the licenses + in this Agreement. + + If Your employer(s) has rights to intellectual property that You + create, You represent that You have received permission to make + the Contributions on behalf of that employer, that Your employer + has waived such rights for Your Contributions, or that Your + employer has executed a separate Corporate Contributor License + Agreement with StrongLoop. + + 5. The Contributions Are Your Original Work + + You represent that each of Your Contributions are Your original + works of authorship (see Section 8 (Submissions on Behalf of + Others) for submission on behalf of others). You represent that to + Your knowledge, no other person claims, or has the right to claim, + any right in any intellectual property right related to Your + Contributions. + + You also represent that You are not legally obligated, whether by + entering into an agreement or otherwise, in any way that conflicts + with the terms of this Agreement. + + You represent that Your Contribution submissions include complete + details of any third-party license or other restriction (including, + but not limited to, related patents and trademarks) of which You + are personally aware and which are associated with any part of + Your Contributions. + + 6. You Don't Have an Obligation to Provide Support for Your Contributions + + You are not expected to provide support for Your Contributions, + except to the extent You desire to provide support. You may provide + support for free, for a fee, or not at all. + + 6. No Warranties or Conditions + + StrongLoop acknowledges that unless required by applicable law or + agreed to in writing, You provide Your Contributions on an "AS IS" + BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER + EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES + OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR + FITNESS FOR A PARTICULAR PURPOSE. + + 7. Submission on Behalf of Others + + If You wish to submit work that is not Your original creation, You + may submit it to StrongLoop separately from any Contribution, + identifying the complete details of its source and of any license + or other restriction (including, but not limited to, related + patents, trademarks, and license agreements) of which You are + personally aware, and conspicuously marking the work as + "Submitted on Behalf of a Third-Party: [named here]". + + 8. Agree to Notify of Change of Circumstances + + You agree to notify StrongLoop of any facts or circumstances of + which You become aware that would make these representations + inaccurate in any respect. Email us at callback@strongloop.com. +``` + +[Google C++ Style Guide]: https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml +[Google Javascript Style Guide]: https://google-styleguide.googlecode.com/svn/trunk/javascriptguide.xml diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..f816ff4 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,7 @@ +strong-build uses a dual license model. + +You may use this library under the terms of the [Artistic 2.0 license][], +or under the terms of the [StrongLoop Subscription Agreement][]. + +[Artistic 2.0 license]: http://opensource.org/licenses/Artistic-2.0 +[StrongLoop Subscription Agreement]: http://strongloop.com/license From b6e34c2038e5aefd213a9f31a5ab724623ab7830 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Tue, 20 May 2014 21:40:43 -0700 Subject: [PATCH 02/23] Add package.json and slb stub --- bin/slb | 3 +++ index.js | 3 +++ package.json | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100755 bin/slb create mode 100644 index.js create mode 100644 package.json diff --git a/bin/slb b/bin/slb new file mode 100755 index 0000000..04446fb --- /dev/null +++ b/bin/slb @@ -0,0 +1,3 @@ +#!/usr/bin/env node + +require('../').build(); diff --git a/index.js b/index.js new file mode 100644 index 0000000..ce4a511 --- /dev/null +++ b/index.js @@ -0,0 +1,3 @@ +exports.build = function build() { + console.log('build!'); +}; diff --git a/package.json b/package.json new file mode 100644 index 0000000..d7040ba --- /dev/null +++ b/package.json @@ -0,0 +1,19 @@ +{ + "name": "strong-build", + "version": "0.0.0", + "repository": { + "type": "git", + "url": "git://github.com/strongloop/strong-build.git" + }, + "author": "engineering@strongloop.com", + "license": { + "name": "Dual MIT/StrongLoop", + "url": "https://github.com/strongloop/strong-build/blob/master/LICENSE" + }, + "readmeFilename": "README.md", + "main": "index.js", + "bin": { + "sl-build": "./bin/slb", + "slb": "./bin/slb" + } +} From f65da5d7c366e936c20cd125aca2ffa8bbe68a98 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Wed, 21 May 2014 15:29:53 -0700 Subject: [PATCH 03/23] Add tap test script --- package.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/package.json b/package.json index d7040ba..490fe64 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,14 @@ }, "readmeFilename": "README.md", "main": "index.js", + "scripts": { + "test": "tap ./test" + }, "bin": { "sl-build": "./bin/slb", "slb": "./bin/slb" + }, + "devDependencies": { + "tap": "^0.4.9" } } From 45468e0b0e0fc06ede976d276578e4981df70d27 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Wed, 21 May 2014 16:00:39 -0700 Subject: [PATCH 04/23] Implement npm install without build scripts Note strong-cli is removed as a dependency, pending a resolution of slc example not actually being installable due to lb versioning. --- .gitignore | 2 ++ bin/slb | 7 ++++++- index.js | 37 ++++++++++++++++++++++++++++++++++-- package.json | 6 +++++- test/test-build.js | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 test/test-build.js diff --git a/.gitignore b/.gitignore index 3c3629e..9196e14 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ node_modules +_suite +npm-debug.log diff --git a/bin/slb b/bin/slb index 04446fb..58e7224 100755 --- a/bin/slb +++ b/bin/slb @@ -1,3 +1,8 @@ #!/usr/bin/env node -require('../').build(); +require('../').build(function(er) { + if (!er) { + exit(0); + } + exit(1); +}); diff --git a/index.js b/index.js index ce4a511..188a894 100644 --- a/index.js +++ b/index.js @@ -1,3 +1,36 @@ -exports.build = function build() { - console.log('build!'); +var debug = require('debug')('strong-build'); +var shell = require('shelljs'); + +var NAME = process.env.SLC_COMMAND ? + 'slc ' + process.env.SLC_COMMAND : + 'slb'; + +function runCommand(cmd, callback) { + debug('run command: %s', cmd); + shell.exec(cmd, {silent: true}, function(code, output) { + debug('code %d: <<<\n%s>>>', code, output); + if (code !== 0) { + var er = Error(cmd); + } + return callback(er, output, code); + }); +} + +function reportRunError(er, output) { + if (!er) return; + + console.error("%s: failed to run `%s`", NAME, er.message); + if (output && output !== '') { + process.stderr.write(output); + } +} + +exports.build = function build(callback) { + runCommand('npm install --ignore-scripts', function(er, output) { + if (er) { + reportRunError(er, output); + return callback(er); + } + return callback(er); + }); }; diff --git a/package.json b/package.json index 490fe64..904dc10 100644 --- a/package.json +++ b/package.json @@ -13,12 +13,16 @@ "readmeFilename": "README.md", "main": "index.js", "scripts": { - "test": "tap ./test" + "test": "tap ./test/test-*.js" }, "bin": { "sl-build": "./bin/slb", "slb": "./bin/slb" }, + "dependencies": { + "debug": "^0.8.1", + "shelljs": "^0.3.0" + }, "devDependencies": { "tap": "^0.4.9" } diff --git a/test/test-build.js b/test/test-build.js new file mode 100644 index 0000000..ecbae07 --- /dev/null +++ b/test/test-build.js @@ -0,0 +1,47 @@ +var assert = require('assert'); +var debug = require('debug')('strong-build:test'); +var path = require('path'); + +require('shelljs/global'); + +var build = require('../'); + +// Check for node silently exiting with code 0 when tests have not passed. +var ok = false; + +process.on('exit', function(code) { + if (code === 0) { + assert(ok); + } +}); + + +debug('test:', 'build slc example'); +debug('cwd:', pwd()); +debug('which slc:', which('slc'), exec('slc -v', {silent: true}).output); + +rm('-rf', '_suite'); +exec('slc example suite _suite --no-install', {silent: true}); +cd('_suite'); +assert(!test('-d', 'node_modules')); + +build.build(function(er) { + debug('built with error?', er); + assert.ifError(er); + assert(test('-d', 'node_modules')); + + var addons = find('node_modules').filter(function(path) { + // .../mongodb/node_modules/bson/ contains compiled addons. Others could in + // the future, but for now, this will assert addons weren't compiled during + // the `npm install` + return path.match(/.*\.node$/) && !path.match(/node_modules\/bson/); + }); + + debug('addons:', addons); + + assert.equal(addons.length, 0); + + ok = true; +}); + +debug('waiting for build...'); From dfa9fda584a94934071a9e5484f08f5ef55e447e Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Wed, 21 May 2014 21:04:47 -0700 Subject: [PATCH 05/23] Implement usage and version options --- index.js | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 188a894..00827ac 100644 --- a/index.js +++ b/index.js @@ -1,9 +1,20 @@ var debug = require('debug')('strong-build'); +var path = require('path'); var shell = require('shelljs'); -var NAME = process.env.SLC_COMMAND ? +var $0 = process.env.SLC_COMMAND ? 'slc ' + process.env.SLC_COMMAND : - 'slb'; + path.basename(process.argv[1]); + +function printHelp($0, prn) { + prn('usage: %s [options]', $0); + prn(''); + prn('Build a node application archive.'); + prn(''); + prn('Options:'); + prn(' -h,--help Print this message and exit.'); + prn(' -v,--version Print version and exit.'); +} function runCommand(cmd, callback) { debug('run command: %s', cmd); @@ -19,13 +30,27 @@ function runCommand(cmd, callback) { function reportRunError(er, output) { if (!er) return; - console.error("%s: failed to run `%s`", NAME, er.message); + console.error("%s: failed to run `%s`", $0, er.message); if (output && output !== '') { process.stderr.write(output); } } exports.build = function build(callback) { + var commandName = process.argv[2]; + + if (['--version', '-v'].indexOf(commandName) != -1) { + console.log(require('./package.json').version); + return callback(); + } + + if (['--help', '-h'].indexOf(commandName) != -1) { + printHelp($0, console.log); + return callback(); + } + + // Ignore unimplemented arguments for now... + runCommand('npm install --ignore-scripts', function(er, output) { if (er) { reportRunError(er, output); From 5ea2f53ca0afcbc385ee2957737aced23e7c8d20 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Wed, 21 May 2014 21:16:39 -0700 Subject: [PATCH 06/23] Implement running of custom package build script --- bin/slb | 4 ++-- index.js | 34 ++++++++++++++++++++++++++-------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/bin/slb b/bin/slb index 58e7224..bb713c3 100755 --- a/bin/slb +++ b/bin/slb @@ -2,7 +2,7 @@ require('../').build(function(er) { if (!er) { - exit(0); + process.exit(0); } - exit(1); + process.exit(1); }); diff --git a/index.js b/index.js index 00827ac..463d1a9 100644 --- a/index.js +++ b/index.js @@ -30,7 +30,7 @@ function runCommand(cmd, callback) { function reportRunError(er, output) { if (!er) return; - console.error("%s: failed to run `%s`", $0, er.message); + console.error("Failed to run `%s`:", er.message); if (output && output !== '') { process.stderr.write(output); } @@ -51,11 +51,29 @@ exports.build = function build(callback) { // Ignore unimplemented arguments for now... - runCommand('npm install --ignore-scripts', function(er, output) { - if (er) { - reportRunError(er, output); - return callback(er); - } - return callback(er); - }); + doNpmInstall(); + + function doNpmInstall() { + runCommand('npm install --ignore-scripts', function(er, output) { + if (er) { + console.error('%s: error during dependency installation', $0); + reportRunError(er, output); + return callback(er); + } + + return doBuildScript(); + }); + } + + function doBuildScript() { + runCommand('npm run build', function(er, output) { + if (er) { + console.error('%s: error in package build script', $0); + reportRunError(er, output); + return callback(er); + } + + return callback(); + }); + } }; From ed8a7df861566ccc616f0ec0d1fddf9719f9f799 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Thu, 22 May 2014 13:08:07 -0700 Subject: [PATCH 07/23] Use option parser, and test option parsing --- bin/slb | 2 +- index.js | 37 +++++++++++++++++++------------ package.json | 6 ++++-- test/test-build.js | 6 +++--- test/test-usage.js | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 85 insertions(+), 20 deletions(-) create mode 100644 test/test-usage.js diff --git a/bin/slb b/bin/slb index bb713c3..0b2504e 100755 --- a/bin/slb +++ b/bin/slb @@ -1,6 +1,6 @@ #!/usr/bin/env node -require('../').build(function(er) { +require('../').build(process.argv, function(er) { if (!er) { process.exit(0); } diff --git a/index.js b/index.js index 463d1a9..47aa0e8 100644 --- a/index.js +++ b/index.js @@ -1,11 +1,8 @@ var debug = require('debug')('strong-build'); +var Parser = require('posix-getopt').BasicParser; var path = require('path'); var shell = require('shelljs'); -var $0 = process.env.SLC_COMMAND ? - 'slc ' + process.env.SLC_COMMAND : - path.basename(process.argv[1]); - function printHelp($0, prn) { prn('usage: %s [options]', $0); prn(''); @@ -36,21 +33,33 @@ function reportRunError(er, output) { } } -exports.build = function build(callback) { - var commandName = process.argv[2]; +exports.build = function build(argv, callback) { + var $0 = process.env.SLC_COMMAND ? + 'slc ' + process.env.SLC_COMMAND : + path.basename(argv[1]); + var parser = new Parser(':v(version)h(help)', argv); + var option; - if (['--version', '-v'].indexOf(commandName) != -1) { - console.log(require('./package.json').version); - return callback(); + while ((option = parser.getopt()) !== undefined) { + switch (option.option) { + case 'v': + console.log(require('./package.json').version); + return callback(); + case 'h': + printHelp($0, console.log); + return callback(); + default: + console.error('Invalid usage (near option \'%s\'), try `%s --help`.', + option.optopt, $0); + return callback(Error('usage')); + } } - if (['--help', '-h'].indexOf(commandName) != -1) { - printHelp($0, console.log); - return callback(); + if (parser.optind() !== argv.length) { + console.error('Invalid usage (extra arguments), try `%s --help`.'); + return callback(Error('usage')); } - // Ignore unimplemented arguments for now... - doNpmInstall(); function doNpmInstall() { diff --git a/package.json b/package.json index 904dc10..55e00fe 100644 --- a/package.json +++ b/package.json @@ -21,9 +21,11 @@ }, "dependencies": { "debug": "^0.8.1", - "shelljs": "^0.3.0" + "shelljs": "^0.3.0", + "posix-getopt": "^1.0.0" }, "devDependencies": { - "tap": "^0.4.9" + "tap": "^0.4.9", + "async": "^0.9.0" } } diff --git a/test/test-build.js b/test/test-build.js index ecbae07..b833091 100644 --- a/test/test-build.js +++ b/test/test-build.js @@ -25,15 +25,15 @@ exec('slc example suite _suite --no-install', {silent: true}); cd('_suite'); assert(!test('-d', 'node_modules')); -build.build(function(er) { +build.build(process.argv, function(er) { debug('built with error?', er); assert.ifError(er); assert(test('-d', 'node_modules')); var addons = find('node_modules').filter(function(path) { // .../mongodb/node_modules/bson/ contains compiled addons. Others could in - // the future, but for now, this will assert addons weren't compiled during - // the `npm install` + // the future, but for now, this works to assert addons weren't compiled + // during the `npm install` return path.match(/.*\.node$/) && !path.match(/node_modules\/bson/); }); diff --git a/test/test-usage.js b/test/test-usage.js new file mode 100644 index 0000000..2db151f --- /dev/null +++ b/test/test-usage.js @@ -0,0 +1,54 @@ +var assert = require('assert'); +var async = require('async'); +var debug = require('debug')('strong-build:test'); +var path = require('path'); + +require('shelljs/global'); + +var build = require('../').build; + +// Check for node silently exiting with code 0 when tests have not passed. +var ok = false; + +process.on('exit', function(code) { + if (code === 0) { + assert(ok); + } +}); + +function expectError(er) { + if(er) { + return null; + } else { + return Error('expected error'); + } +} + +// argv [0] and [1] are ignored (they are node and script name, not options) +async.parallel([ + build.bind(null, ['', '', '-h']), + build.bind(null, ['', '', '--help']), + build.bind(null, ['', '', '-hv']), + build.bind(null, ['', '', '-v']), + build.bind(null, ['', '', '--version']), + build.bind(null, ['', '', '-vh']), + function(callback) { + build(['', '', 'no-such-arg'], function(er) { + return callback(expectError(er)); + }); + }, + function(callback) { + build(['', '', '--no-such-option'], function(er) { + return callback(expectError(er)); + }); + }, + function(callback) { + build(['', '', '-Z'], function(er) { + return callback(expectError(er)); + }); + }, +], function(er, results) { + debug('test-help: error=%s:', er, results); + assert.ifError(er); + ok = true; +}); From b64f480ef18f426cd5dea4aae2988e78ba0b3d33 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Thu, 22 May 2014 14:16:58 -0700 Subject: [PATCH 08/23] Refactor build test setup so it can be reused --- test/build-example.js | 39 +++++++++++++++++++++++++++++++++++++++ test/test-build.js | 34 ++-------------------------------- 2 files changed, 41 insertions(+), 32 deletions(-) create mode 100644 test/build-example.js diff --git a/test/build-example.js b/test/build-example.js new file mode 100644 index 0000000..90cfbfd --- /dev/null +++ b/test/build-example.js @@ -0,0 +1,39 @@ +module.exports = function buildExample(args, callback) { + assert = require('assert'); + debug = require('debug')('strong-build:test'); + var path = require('path'); + + require('shelljs/global'); + + var build = require('../'); + + // Check for node silently exiting with code 0 when tests have not passed. + var ok = false; + + process.on('exit', function(code) { + if (code === 0) { + assert(ok); + } + }); + + + debug('test: build slc example, args:', args); + debug('cwd:', pwd()); + debug('which slc:', which('slc'), exec('slc -v', {silent: true}).output); + + rm('-rf', '_suite'); + exec('slc example suite _suite --no-install', {silent: true}); + cd('_suite'); + assert(!test('-d', 'node_modules')); + + var argv = ['node', 'slb'].concat(args); + + debug('build with argv:', argv); + + build.build(argv, function(er) { + ok = true; + return callback(er) + }); + + debug('waiting for build...'); +}; diff --git a/test/test-build.js b/test/test-build.js index b833091..8c1c339 100644 --- a/test/test-build.js +++ b/test/test-build.js @@ -1,31 +1,5 @@ -var assert = require('assert'); -var debug = require('debug')('strong-build:test'); -var path = require('path'); - -require('shelljs/global'); - -var build = require('../'); - -// Check for node silently exiting with code 0 when tests have not passed. -var ok = false; - -process.on('exit', function(code) { - if (code === 0) { - assert(ok); - } -}); - - -debug('test:', 'build slc example'); -debug('cwd:', pwd()); -debug('which slc:', which('slc'), exec('slc -v', {silent: true}).output); - -rm('-rf', '_suite'); -exec('slc example suite _suite --no-install', {silent: true}); -cd('_suite'); -assert(!test('-d', 'node_modules')); - -build.build(process.argv, function(er) { +// build example with --install +require('./build-example')(['-i'], function(er) { debug('built with error?', er); assert.ifError(er); assert(test('-d', 'node_modules')); @@ -40,8 +14,4 @@ build.build(process.argv, function(er) { debug('addons:', addons); assert.equal(addons.length, 0); - - ok = true; }); - -debug('waiting for build...'); From 36f190cc3e1f299b4397da711619f98ad845e1f5 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Thu, 22 May 2014 14:45:45 -0700 Subject: [PATCH 09/23] Implement --scripts, to build binary addons --- index.js | 53 +++++++++++++++++++++++++++++--------- package.json | 5 ++-- test/test-build-scripts.js | 17 ++++++++++++ 3 files changed, 61 insertions(+), 14 deletions(-) create mode 100644 test/test-build-scripts.js diff --git a/index.js b/index.js index 47aa0e8..9da8260 100644 --- a/index.js +++ b/index.js @@ -1,16 +1,24 @@ +var assert = require('assert'); var debug = require('debug')('strong-build'); var Parser = require('posix-getopt').BasicParser; var path = require('path'); var shell = require('shelljs'); +var vasync = require('vasync'); function printHelp($0, prn) { prn('usage: %s [options]', $0); prn(''); prn('Build a node application archive.'); prn(''); + prn('Archives are built without running scripts (by default) to avoid'); + prn('compiling any binary addons. Build and install scripts should be run'); + prn('on the deployment server using `npm rebuild; npm install`.'); + prn(''); prn('Options:'); - prn(' -h,--help Print this message and exit.'); - prn(' -v,--version Print version and exit.'); + prn(' -h,--help Print this message and exit.'); + prn(' -v,--version Print version and exit.'); + prn(' -i,--install Install dependencies (without scripts, by default).'); + prn(' --scripts If installing, run scripts (to build addons).'); } function runCommand(cmd, callback) { @@ -37,8 +45,12 @@ exports.build = function build(argv, callback) { var $0 = process.env.SLC_COMMAND ? 'slc ' + process.env.SLC_COMMAND : path.basename(argv[1]); - var parser = new Parser(':v(version)h(help)', argv); + var parser = new Parser( + ':v(version)h(help)s(scripts)i(install)', + argv); var option; + var install; + var scripts; while ((option = parser.getopt()) !== undefined) { switch (option.option) { @@ -48,6 +60,12 @@ exports.build = function build(argv, callback) { case 'h': printHelp($0, console.log); return callback(); + case 's': + scripts = true; + break; + case 'i': + install = true; + break; default: console.error('Invalid usage (near option \'%s\'), try `%s --help`.', option.optopt, $0); @@ -60,29 +78,40 @@ exports.build = function build(argv, callback) { return callback(Error('usage')); } - doNpmInstall(); + if (!install && !pack) { + install = pack = true; + } + + var steps = []; + + if (install) { + steps.push(doNpmInstall); + } + + vasync.pipeline({funcs: steps}, callback); - function doNpmInstall() { - runCommand('npm install --ignore-scripts', function(er, output) { + function doNpmInstall(_, callback) { + var npmInstall = 'npm install'; + if (!scripts) { + npmInstall += ' --ignore-scripts'; + } + runCommand(npmInstall, function(er, output) { if (er) { console.error('%s: error during dependency installation', $0); reportRunError(er, output); return callback(er); } - - return doBuildScript(); + return doBuildScript(_, callback); }); } - function doBuildScript() { + function doBuildScript(_, callback) { runCommand('npm run build', function(er, output) { if (er) { console.error('%s: error in package build script', $0); reportRunError(er, output); - return callback(er); } - - return callback(); + return callback(er); }); } }; diff --git a/package.json b/package.json index 55e00fe..33f44d3 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "readmeFilename": "README.md", "main": "index.js", "scripts": { - "test": "tap ./test/test-*.js" + "test": "tap --timeout=300 ./test/test-*.js" }, "bin": { "sl-build": "./bin/slb", @@ -22,7 +22,8 @@ "dependencies": { "debug": "^0.8.1", "shelljs": "^0.3.0", - "posix-getopt": "^1.0.0" + "posix-getopt": "^1.0.0", + "vasync": "^1.4.3" }, "devDependencies": { "tap": "^0.4.9", diff --git a/test/test-build-scripts.js b/test/test-build-scripts.js new file mode 100644 index 0000000..c0aac52 --- /dev/null +++ b/test/test-build-scripts.js @@ -0,0 +1,17 @@ +// build example with --install +require('./build-example')(['-i', '--scripts'], function(er) { + debug('built with error?', er); + assert.ifError(er); + assert(test('-d', 'node_modules')); + + var addons = find('node_modules').filter(function(path) { + // .../mongodb/node_modules/bson/ contains compiled addons. Others could in + // the future, but for now, this works to assert addons weren't compiled + // during the `npm install` + return path.match(/.*\.node$/) && !path.match(/node_modules\/bson/); + }); + + debug('addons:', addons); + + assert(addons.length > 0); +}); From 8211314b62de51da2241e30d65325b24ca33acbb Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Thu, 22 May 2014 15:08:50 -0700 Subject: [PATCH 10/23] Implement --pack, to create an archive --- index.js | 25 ++++++++++++++++++- test/build-example.js | 3 ++- test/{test-build.js => test-build-install.js} | 1 - test/test-build-pack.js | 11 ++++++++ test/test-build-scripts.js | 1 - 5 files changed, 37 insertions(+), 4 deletions(-) rename test/{test-build.js => test-build-install.js} (94%) create mode 100644 test/test-build-pack.js diff --git a/index.js b/index.js index 9da8260..fad2b4b 100644 --- a/index.js +++ b/index.js @@ -14,11 +14,15 @@ function printHelp($0, prn) { prn('compiling any binary addons. Build and install scripts should be run'); prn('on the deployment server using `npm rebuild; npm install`.'); prn(''); + prn('Pack output is a tar file in the format produced by `npm pack` and'); + prn('accepted by `npm install`.'); + prn(''); prn('Options:'); prn(' -h,--help Print this message and exit.'); prn(' -v,--version Print version and exit.'); prn(' -i,--install Install dependencies (without scripts, by default).'); prn(' --scripts If installing, run scripts (to build addons).'); + prn(' -p,--pack Pack into a publishable archive (with dependencies)'); } function runCommand(cmd, callback) { @@ -46,11 +50,13 @@ exports.build = function build(argv, callback) { 'slc ' + process.env.SLC_COMMAND : path.basename(argv[1]); var parser = new Parser( - ':v(version)h(help)s(scripts)i(install)', + ':v(version)h(help)s(scripts)i(install)p(pack)', argv); var option; var install; var scripts; + var pack; + while ((option = parser.getopt()) !== undefined) { switch (option.option) { @@ -66,6 +72,9 @@ exports.build = function build(argv, callback) { case 'i': install = true; break; + case 'p': + pack = true; + break; default: console.error('Invalid usage (near option \'%s\'), try `%s --help`.', option.optopt, $0); @@ -88,6 +97,10 @@ exports.build = function build(argv, callback) { steps.push(doNpmInstall); } + if (pack) { + steps.push(doNpmPack); + } + vasync.pipeline({funcs: steps}, callback); function doNpmInstall(_, callback) { @@ -114,4 +127,14 @@ exports.build = function build(argv, callback) { return callback(er); }); } + + function doNpmPack(_, callback) { + runCommand('npm pack', function(er, output) { + if (er) { + console.error('%s: error packing an archive', $0); + reportRunError(er, output); + } + return callback(er); + }); + } }; diff --git a/test/build-example.js b/test/build-example.js index 90cfbfd..cca4f9e 100644 --- a/test/build-example.js +++ b/test/build-example.js @@ -1,7 +1,8 @@ module.exports = function buildExample(args, callback) { assert = require('assert'); debug = require('debug')('strong-build:test'); - var path = require('path'); + path = require('path'); + util = require('util'); require('shelljs/global'); diff --git a/test/test-build.js b/test/test-build-install.js similarity index 94% rename from test/test-build.js rename to test/test-build-install.js index 8c1c339..8085d91 100644 --- a/test/test-build.js +++ b/test/test-build-install.js @@ -1,4 +1,3 @@ -// build example with --install require('./build-example')(['-i'], function(er) { debug('built with error?', er); assert.ifError(er); diff --git a/test/test-build-pack.js b/test/test-build-pack.js new file mode 100644 index 0000000..1ff0bd8 --- /dev/null +++ b/test/test-build-pack.js @@ -0,0 +1,11 @@ +require('./build-example')(['--install', '-p'], function(er) { + debug('built with error?', er); + assert.ifError(er); + assert(test('-d', 'node_modules')); + var info = require(path.resolve('package.json')); + assert.equal(info.name, 'sls-sample-app'); + var tgz = util.format('%s-%s.tgz', info.name, info.version); + assert(test('-f', tgz), 'expected to find ' + tgz); + + // XXX(sam) open the tgz, and assert something about its contents? +}); diff --git a/test/test-build-scripts.js b/test/test-build-scripts.js index c0aac52..7a9afd5 100644 --- a/test/test-build-scripts.js +++ b/test/test-build-scripts.js @@ -1,4 +1,3 @@ -// build example with --install require('./build-example')(['-i', '--scripts'], function(er) { debug('built with error?', er); assert.ifError(er); From a3f4fca67e0035f220b876c196fe867ba49007af Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Fri, 23 May 2014 12:42:47 -0700 Subject: [PATCH 11/23] Implement --bundle, so dependencies are packed --- index.js | 84 ++++++++++++++++++++++++++++++++++---- package.json | 7 +++- test/build-example.js | 23 +++++++++++ test/test-build-bundled.js | 36 ++++++++++++++++ test/test-build-pack.js | 9 +++- 5 files changed, 149 insertions(+), 10 deletions(-) create mode 100644 test/test-build-bundled.js diff --git a/index.js b/index.js index fad2b4b..ae856df 100644 --- a/index.js +++ b/index.js @@ -1,19 +1,25 @@ var assert = require('assert'); var debug = require('debug')('strong-build'); +var fs = require('fs'); var Parser = require('posix-getopt').BasicParser; var path = require('path'); +var json = require('json-file-plus'); +var lodash = require('lodash'); var shell = require('shelljs'); var vasync = require('vasync'); function printHelp($0, prn) { prn('usage: %s [options]', $0); prn(''); - prn('Build a node application archive.'); + prn('Build a node application package.'); prn(''); - prn('Archives are built without running scripts (by default) to avoid'); + prn('Packages are built without running scripts (by default) to avoid'); prn('compiling any binary addons. Build and install scripts should be run'); prn('on the deployment server using `npm rebuild; npm install`.'); prn(''); + prn('... XXX(SR) I think --bundle should be default behaviour of --install'); + prn('... XXX(SR) and there should be --no-bundle (like --scripts) to avoid'); + prn(''); prn('Pack output is a tar file in the format produced by `npm pack` and'); prn('accepted by `npm install`.'); prn(''); @@ -22,7 +28,8 @@ function printHelp($0, prn) { prn(' -v,--version Print version and exit.'); prn(' -i,--install Install dependencies (without scripts, by default).'); prn(' --scripts If installing, run scripts (to build addons).'); - prn(' -p,--pack Pack into a publishable archive (with dependencies)'); + prn(' -b,--bundle Modify package to bundle deployment dependencies.'); + prn(' -p,--pack Pack into a publishable archive (with dependencies).'); } function runCommand(cmd, callback) { @@ -50,14 +57,14 @@ exports.build = function build(argv, callback) { 'slc ' + process.env.SLC_COMMAND : path.basename(argv[1]); var parser = new Parser( - ':v(version)h(help)s(scripts)i(install)p(pack)', + ':v(version)h(help)s(scripts)i(install)b(bundle)p(pack)', argv); var option; var install; var scripts; + var bundle; var pack; - while ((option = parser.getopt()) !== undefined) { switch (option.option) { case 'v': @@ -72,6 +79,9 @@ exports.build = function build(argv, callback) { case 'i': install = true; break; + case 'b': + bundle = true; + break; case 'p': pack = true; break; @@ -87,8 +97,8 @@ exports.build = function build(argv, callback) { return callback(Error('usage')); } - if (!install && !pack) { - install = pack = true; + if (!install && !bundle && !pack) { + install = bundle = pack = true; } var steps = []; @@ -97,6 +107,10 @@ exports.build = function build(argv, callback) { steps.push(doNpmInstall); } + if (bundle) { + steps.push(doBundle); + } + if (pack) { steps.push(doNpmPack); } @@ -128,6 +142,62 @@ exports.build = function build(argv, callback) { }); } + function doBundle(_, callback) { + // Build output won't get packed if it is .npmignored (a configuration + // error, don't .npmignore your build output) or if there is no .npmignore, + // if it is .gitignored (as they should be). So, create an empty .npmignore + // if there is a .gitignore but not a .npmignore so build products are + // packed. + if (fs.existsSync('.gitignore')) { + fs.close(fs.openSync('.npmignore', 'a')); + } + + // node_modules is unconditionally ignored by npm pack, the only way to get + // the dependencies packed is to name them in the package.json's + // bundledDepenencies. + var deps = fs.readdirSync('node_modules').filter(function(file) { + // Only directories containing a package.json are packages. + return shell.test( + '-f', + path.join('node_modules', file, 'package.json') + ); + }); + + var info = require(path.resolve('package.json')); + var dev = Object.keys(info.devDependencies || {}); + + // Remove dev dependencies, bundle any others, including manually + // installed, deps comitted into version control, optional, etc. + var bundle = lodash.difference(deps, dev); + + // Two names are allowed for this key... use 'bundledDependencies' unless + // package is already using the other name. + var key = info.bundleDependencies ? + 'bundleDependencies' : + 'bundledDependencies'; + + bundle = lodash.uniq(bundle.concat( + info.bundleDependencies || info.bundledDependencies || [] + )); + + // Re-write package.json, preserving its format if possible. + json('package.json', function(er, p) { + if (er) { + console.error('%s: error reading package.json: %s', $0, er.message); + return callback(er); + } + + p.data[key] = bundle; + + p.save(function(er) { + if (er) { + console.error('%s: error writing package.json: %s', $0, er.message); + } + return callback(er); + }); + }); + } + function doNpmPack(_, callback) { runCommand('npm pack', function(er, output) { if (er) { diff --git a/package.json b/package.json index 33f44d3..9f18e33 100644 --- a/package.json +++ b/package.json @@ -23,10 +23,13 @@ "debug": "^0.8.1", "shelljs": "^0.3.0", "posix-getopt": "^1.0.0", - "vasync": "^1.4.3" + "vasync": "^1.4.3", + "json-file-plus": "^0.2.3", + "lodash": "^2.4.1" }, "devDependencies": { "tap": "^0.4.9", - "async": "^0.9.0" + "async": "^0.9.0", + "tar": "^0.1.19" } } diff --git a/test/build-example.js b/test/build-example.js index cca4f9e..0da89b6 100644 --- a/test/build-example.js +++ b/test/build-example.js @@ -1,8 +1,31 @@ module.exports = function buildExample(args, callback) { assert = require('assert'); debug = require('debug')('strong-build:test'); + fs = require('fs'); path = require('path'); + tar = require('tar'); util = require('util'); + zlib = require('zlib'); + + tar.list = function list(tarfile, callback) { + var paths = []; + + fs.createReadStream(tarfile) + .on('error', callback) + .pipe(zlib.Unzip()) + .pipe(tar.Parse()) + .on('entry', function(entry) { + paths.push(entry.path); + }) + .on('end', function() { + debug('tarfile %s list:', tarfile, paths); + return callback(null, paths); + }); + }; + + fs.readJsonSync = function readJsonSync(file) { + return JSON.parse(fs.readFileSync(file)); + }; require('shelljs/global'); diff --git a/test/test-build-bundled.js b/test/test-build-bundled.js new file mode 100644 index 0000000..7319ed8 --- /dev/null +++ b/test/test-build-bundled.js @@ -0,0 +1,36 @@ +var lodash = require('lodash'); +require('./build-example')(['--install', '-p', '--bundle'], function(er) { + assert.ifError(er); + var info = fs.readJsonSync('package.json'); + var tgz = util.format('%s-%s.tgz', info.name, info.version); + + // loopback should be bundled (normal dep) + // loopback-connector-mysql should be bundled (optional dep) + // mocha should not (dev dep) + function isBundled(bundled, name) { + return lodash.findIndex(bundled, function(_) { + return _ === name; + }) >= 0; + } + debug('package bundles:', info.bundledDependencies); + assert(isBundled(info.bundledDependencies, 'loopback')); + assert(isBundled(info.bundledDependencies, 'loopback-connector-mysql')); + assert(!isBundled(info.bundledDependencies, 'mocha')); + + tar.list(tgz, function(er, paths) { + debug('tarfile %s bundles:', tgz, bundled); + var bundled = lodash(paths).map(function(_) { + // We are looking for the top-level bundled deps, they look like + // package/node_modules/shuffle/package.json + var components = _.split(path.sep); + + if (components.length === 4 && components[3] === 'package.json') { + return components[2]; + } + return undefined; + }).compact().sort().value(); + debug('tarfile %s bundles:', tgz, bundled); + debug('should be:', info.bundledDependencies); + assert.deepEqual(bundled, info.bundledDependencies); + }); +}); diff --git a/test/test-build-pack.js b/test/test-build-pack.js index 1ff0bd8..906199b 100644 --- a/test/test-build-pack.js +++ b/test/test-build-pack.js @@ -7,5 +7,12 @@ require('./build-example')(['--install', '-p'], function(er) { var tgz = util.format('%s-%s.tgz', info.name, info.version); assert(test('-f', tgz), 'expected to find ' + tgz); - // XXX(sam) open the tgz, and assert something about its contents? + tar.list(tgz, function(er, paths) { + var bundled = paths.filter(function(path) { + return path.match(/node_modules/); + }); + debug('tarfile %s bundles:', tgz, bundled); + + assert.equal(bundled.length, 0, 'bundling not requested'); + }); }); From 8123af772cdb7c573820799f2c46424573888bb4 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Fri, 23 May 2014 13:48:30 -0700 Subject: [PATCH 12/23] test: use strong-cli as a dev dependency `slc example` has been fixed so it now can be `npm install`ed, so use it as a dev dependency. --- test/build-example.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/build-example.js b/test/build-example.js index 0da89b6..726f9c1 100644 --- a/test/build-example.js +++ b/test/build-example.js @@ -41,9 +41,12 @@ module.exports = function buildExample(args, callback) { }); - debug('test: build slc example, args:', args); - debug('cwd:', pwd()); - debug('which slc:', which('slc'), exec('slc -v', {silent: true}).output); + console.log('build example with %j in %s with %s at %s', + args, + pwd(), + which('slc'), + exec('slc -v', {silent: true}).output.split('\n')[0].split(' ')[1] + ); rm('-rf', '_suite'); exec('slc example suite _suite --no-install', {silent: true}); From e7afc8eb80253a9b1a843a70397c3b494878cda3 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Fri, 23 May 2014 14:14:27 -0700 Subject: [PATCH 13/23] pack: place archive in parent directory Current build directory may be temporary, get git cleaned, and the pack output might itself get packed. Avoid this mess by putting the pack file one directory up. --- .gitignore | 3 ++- index.js | 10 +++++++++- test/test-build-bundled.js | 2 +- test/test-build-pack.js | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 9196e14..987276f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules -_suite npm-debug.log +sls-sample-app-2.0.3.tgz +_suite diff --git a/index.js b/index.js index ae856df..b41c50b 100644 --- a/index.js +++ b/index.js @@ -203,8 +203,16 @@ exports.build = function build(argv, callback) { if (er) { console.error('%s: error packing an archive', $0); reportRunError(er, output); + return callback(er); } - return callback(er); + + // npm pack output is a single line with the pack file name + var src = output.split('\n')[0]; + var dst = path.join('..', src); + + shell.mv('-f', src, dst); + + return callback(); }); } }; diff --git a/test/test-build-bundled.js b/test/test-build-bundled.js index 7319ed8..71a1065 100644 --- a/test/test-build-bundled.js +++ b/test/test-build-bundled.js @@ -2,7 +2,7 @@ var lodash = require('lodash'); require('./build-example')(['--install', '-p', '--bundle'], function(er) { assert.ifError(er); var info = fs.readJsonSync('package.json'); - var tgz = util.format('%s-%s.tgz', info.name, info.version); + var tgz = path.join('..', util.format('%s-%s.tgz', info.name, info.version)); // loopback should be bundled (normal dep) // loopback-connector-mysql should be bundled (optional dep) diff --git a/test/test-build-pack.js b/test/test-build-pack.js index 906199b..2a43aa1 100644 --- a/test/test-build-pack.js +++ b/test/test-build-pack.js @@ -4,7 +4,7 @@ require('./build-example')(['--install', '-p'], function(er) { assert(test('-d', 'node_modules')); var info = require(path.resolve('package.json')); assert.equal(info.name, 'sls-sample-app'); - var tgz = util.format('%s-%s.tgz', info.name, info.version); + var tgz = path.join('..', util.format('%s-%s.tgz', info.name, info.version)); assert(test('-f', tgz), 'expected to find ' + tgz); tar.list(tgz, function(er, paths) { From 0c93daa5740c1fdb26cfbda2a697c80abd950f5c Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Fri, 23 May 2014 14:47:30 -0700 Subject: [PATCH 14/23] doc: add README.md and update --help output --- README.md | 32 ++++++++++++++++++++++++++++++++ index.js | 18 +++++++++++++----- 2 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..e3578e7 --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# strong-build + +usage: `slb [options]` + +Build a node application package. + +Packages are built without running scripts (by default, see --scripts) +to avoid compiling any binary addons. Compilation and install scripts +should be run on the deployment server using: + + npm rebuild + npm install + +Custom build steps, such as `grunt build`, can be specified as a +'build' script in the package.json's `scripts` property and will +be run after installing dependencies. + +Bundling configures the package.json and .npmignore so deployment (not +development) dependencies as well as any 'build' script output will +not be ignored by `npm pack`. + +Pack output is a tar file in the format produced by `npm pack` and +accepted by `npm install`. + +Options: + +* -h,--help Print this message and exit. +* -v,--version Print version and exit. +* -i,--install Install dependencies (without scripts, by default). +* --scripts If installing, run scripts (to build addons). +* -b,--bundle Modify package to bundle deployment dependencies. +* -p,--pack Pack into a publishable archive (with dependencies). diff --git a/index.js b/index.js index b41c50b..86193ef 100644 --- a/index.js +++ b/index.js @@ -13,12 +13,20 @@ function printHelp($0, prn) { prn(''); prn('Build a node application package.'); prn(''); - prn('Packages are built without running scripts (by default) to avoid'); - prn('compiling any binary addons. Build and install scripts should be run'); - prn('on the deployment server using `npm rebuild; npm install`.'); + prn('Packages are built without running scripts (by default, see --scripts)'); + prn('to avoid compiling any binary addons. Compilation and install scripts'); + prn('should be run on the deployment server using:'); prn(''); - prn('... XXX(SR) I think --bundle should be default behaviour of --install'); - prn('... XXX(SR) and there should be --no-bundle (like --scripts) to avoid'); + prn(' npm rebuild'); + prn(' npm install'); + prn(''); + prn('Custom build steps, such as `grunt build`, can be specified as a'); + prn('\'build\' script in the package.json\'s `scripts` property and will'); + prn('be run after installing dependencies.'); + prn(''); + prn('Bundling configures the package.json and .npmignore so deployment (not'); + prn('development) dependencies as well as any \'build\' script output will'); + prn('not be ignored by `npm pack`.'); prn(''); prn('Pack output is a tar file in the format produced by `npm pack` and'); prn('accepted by `npm install`.'); From 0530f3eb70dbb7b9b71c6ce20e9f0c6d0ab8b548 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Mon, 26 May 2014 15:44:10 -0700 Subject: [PATCH 15/23] onto: commit source tree onto a deploy branch --- .gitignore | 1 + README.md | 8 ++++- index.js | 38 ++++++++++++++++++++---- lib/git.js | 75 +++++++++++++++++++++++++++++++++++++++++++++++ test/test-onto.js | 75 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 191 insertions(+), 6 deletions(-) create mode 100644 lib/git.js create mode 100644 test/test-onto.js diff --git a/.gitignore b/.gitignore index 987276f..0b70e9c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ node_modules npm-debug.log sls-sample-app-2.0.3.tgz _suite +_onto diff --git a/README.md b/README.md index e3578e7..8f047a1 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,12 @@ usage: `slb [options]` Build a node application package. +With no options, the default is to install, bundle, and pack. + +When using git, the current branch can be committed onto a deployment +branch before the build. This allows the build products to be +committed and synchronized with a source-only branch. + Packages are built without running scripts (by default, see --scripts) to avoid compiling any binary addons. Compilation and install scripts should be run on the deployment server using: @@ -23,9 +29,9 @@ Pack output is a tar file in the format produced by `npm pack` and accepted by `npm install`. Options: - * -h,--help Print this message and exit. * -v,--version Print version and exit. +* -onto BRANCH Merge git HEAD to deployment BRANCH. * -i,--install Install dependencies (without scripts, by default). * --scripts If installing, run scripts (to build addons). * -b,--bundle Modify package to bundle deployment dependencies. diff --git a/index.js b/index.js index 86193ef..91ee065 100644 --- a/index.js +++ b/index.js @@ -1,10 +1,11 @@ var assert = require('assert'); var debug = require('debug')('strong-build'); var fs = require('fs'); -var Parser = require('posix-getopt').BasicParser; -var path = require('path'); +var git = require('./lib/git'); var json = require('json-file-plus'); var lodash = require('lodash'); +var Parser = require('posix-getopt').BasicParser; +var path = require('path'); var shell = require('shelljs'); var vasync = require('vasync'); @@ -13,6 +14,12 @@ function printHelp($0, prn) { prn(''); prn('Build a node application package.'); prn(''); + prn('With no options, the default is to install, bundle, and pack.'); + prn(''); + prn('When using git, the current branch can be committed onto a deployment'); + prn('branch before the build. This allows the build products to be'); + prn('committed and synchronized with a source-only branch.'); + prn(''); prn('Packages are built without running scripts (by default, see --scripts)'); prn('to avoid compiling any binary addons. Compilation and install scripts'); prn('should be run on the deployment server using:'); @@ -34,6 +41,7 @@ function printHelp($0, prn) { prn('Options:'); prn(' -h,--help Print this message and exit.'); prn(' -v,--version Print version and exit.'); + prn(' -onto BRANCH Merge git HEAD to deployment BRANCH.'); prn(' -i,--install Install dependencies (without scripts, by default).'); prn(' --scripts If installing, run scripts (to build addons).'); prn(' -b,--bundle Modify package to bundle deployment dependencies.'); @@ -65,9 +73,10 @@ exports.build = function build(argv, callback) { 'slc ' + process.env.SLC_COMMAND : path.basename(argv[1]); var parser = new Parser( - ':v(version)h(help)s(scripts)i(install)b(bundle)p(pack)', + ':v(version)h(help)s(scripts)i(install)b(bundle)p(pack)O:(onto)', argv); var option; + var onto; var install; var scripts; var bundle; @@ -93,6 +102,9 @@ exports.build = function build(argv, callback) { case 'p': pack = true; break; + case 'O': + onto = option.optarg; + break; default: console.error('Invalid usage (near option \'%s\'), try `%s --help`.', option.optopt, $0); @@ -105,12 +117,16 @@ exports.build = function build(argv, callback) { return callback(Error('usage')); } - if (!install && !bundle && !pack) { + if (!onto && !install && !bundle && !pack) { install = bundle = pack = true; } var steps = []; + if (onto) { + steps.push(doGitOnto); + } + if (install) { steps.push(doNpmInstall); } @@ -125,6 +141,18 @@ exports.build = function build(argv, callback) { vasync.pipeline({funcs: steps}, callback); + function doGitOnto(_, callback) { + try { + var info = git.onto($0, onto); + console.log('%s: merged `%s` onto `%s`, ready to build', + $0, info.srcBranch, info.dstBranch); + return callback(); + } catch(er) { + console.error('%s: %s', $0, er.message); + return callback(er); + } + } + function doNpmInstall(_, callback) { var npmInstall = 'npm install'; if (!scripts) { @@ -175,7 +203,7 @@ exports.build = function build(argv, callback) { var dev = Object.keys(info.devDependencies || {}); // Remove dev dependencies, bundle any others, including manually - // installed, deps comitted into version control, optional, etc. + // installed, deps committed into version control, optional, etc. var bundle = lodash.difference(deps, dev); // Two names are allowed for this key... use 'bundledDependencies' unless diff --git a/lib/git.js b/lib/git.js new file mode 100644 index 0000000..d6175f1 --- /dev/null +++ b/lib/git.js @@ -0,0 +1,75 @@ +var assert = require('assert'); +var debug = require('debug')('strong-build'); +var fs = require('fs'); +var lodash = require('lodash'); +var path = require('path'); +var shell = require('shelljs'); +var util = require('util'); + +function fmt() { + return util.format.apply(util, arguments); +} + +function execSync(cmd, error) { + var exec = shell.exec(cmd, {silent: true}); + + debug('exec `%s`: ...', cmd); + + exec.output = exec.output.trim(); + + debug('exec `%s`: (code %s) %s', cmd, exec.code, exec.output); + + if (exec.code !== 0) { + throw Error(error); + } + + return exec.output; +} + +exports.onto = function gitOnto($0, gitDstBranch) { + // Get name of current branch, and the tree SHA of its HEAD commit + var gitSrcBranch = execSync( + 'git symbolic-ref --short HEAD', + 'failed to parse branch `HEAD`, is this a git repository?' + ); + + var gitSrcTreeSha = execSync( + 'git log -1 --pretty=format:"%t" HEAD', + 'failed to parse branch `HEAD`, is this a git repository?' + ); + + // Get ref name of destination branch, to validate it exists, and because + // we will use it later. + var dstRefInfo = execSync( + fmt('git show-ref --heads --abbrev "%s"', gitDstBranch), + fmt('failed to parse branch `%s`, does it exist?', gitDstBranch) + ); + var dstRef = dstRefInfo.split(' ')[1]; + + // Commit the source tree to the head of the destination branch, and update + // the destination branch ref. + var message = fmt('Build `%s` onto `%s`', gitSrcBranch, gitDstBranch); + var commitSha = execSync( + fmt('git commit-tree -p HEAD -p "%s" -m \'%s\' %s', + gitDstBranch, message, gitSrcTreeSha), + fmt('failed to commit `%s` onto `%s`', gitSrcBranch, gitDstBranch) + ); + + execSync( + fmt('git update-ref "%s" %s', dstRef, commitSha), + fmt('failed to merge `%s` onto `%s`', gitSrcBranch, gitDstBranch) + ); + + // Checkout the destination branch so that we are ready to build + execSync( + fmt('git checkout "%s"', gitDstBranch), + fmt('failed to checkout `%s`', gitDstBranch) + ); + + return { + dstBranch: gitDstBranch, + srcBranch: gitSrcBranch, + srcTreeSha: gitSrcTreeSha, + commitSha: commitSha, + }; +} diff --git a/test/test-onto.js b/test/test-onto.js new file mode 100644 index 0000000..4bbeafd --- /dev/null +++ b/test/test-onto.js @@ -0,0 +1,75 @@ +var assert = require('assert'); +var debug = require('debug')('strong-build:test'); +var fs = require('fs'); +var path = require('path'); +var tar = require('tar'); +var util = require('util'); +var zlib = require('zlib'); + +var shell = require('shelljs/global'); + +// Check for node silently exiting with code 0 when tests have not passed. +var ok = false; + +process.on('exit', function(code) { + if (code === 0) { + assert(ok); + } +}); + +console.log('pwd: %s', pwd()); + +// Create two branches that are different, and commit the src +// over the dst, and then diff them to prove identical. +rm('-rf', '_onto'); +mkdir('_onto'); +cd('_onto'); +run('git init'); +touchAndCommit('.gitignore'); +run('git checkout -b dst master'); +touchAndCommit('only-on-dst'); +run('git checkout -b src master'); +touchAndCommit('only-on-src'); + +assert(!branchEqual('src', 'dst')); + +var build = require('../'); + +build.build(['node', 'main.js', '--onto=dst'], function(er) { + assert.ifError(er); + assert(branchEqual('src', 'dst')); + ok = true; +}); + +// Shell wrappers +function run() { + var cmd = util.format.apply(util, arguments); + debug('exec `%s`: ...', cmd); + var out = exec(cmd, {silent: true}); + out.output = out.output.trim(); + console.log('exec `%s`: (code %s)\n%s', cmd, out.code, out.output); + if (out.code !== 0) { + throw Error(error); + } + return out.output; +} + +function touch(name) { + console.log('touch %s', name); + fs.closeSync(fs.openSync(name, 'a')); +} + +function touchAndCommit(name) { + touch(name); + run('git add %s', name); + run('git commit -m add-%s', name); +} + +function branchEqual(src, dst) { + try { + run('git diff --quiet %s %s', src, dst); + return true; + } catch(er) { + return false; + } +} From a0ab12cd10e0ad59c7ee9c31b48fe505c40e23f0 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Tue, 27 May 2014 15:55:19 -0700 Subject: [PATCH 16/23] bundle: warn on problematic or incomplete config A missing .npmignore when .gitignore is present, or both bundleDependencies and bundledDependencies in package.json are problematic. --- index.js | 37 +++++++++++++++++++++++++------------ test/test-build-bundled.js | 12 ++++++------ 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/index.js b/index.js index 91ee065..f9a40ba 100644 --- a/index.js +++ b/index.js @@ -33,7 +33,9 @@ function printHelp($0, prn) { prn(''); prn('Bundling configures the package.json and .npmignore so deployment (not'); prn('development) dependencies as well as any \'build\' script output will'); - prn('not be ignored by `npm pack`.'); + prn('not be ignored by `npm pack`. If an .npmignore is created, check it'); + prn('carefully to ensure build products are packed, but editor and test'); + prn('ephemera are not.'); prn(''); prn('Pack output is a tar file in the format produced by `npm pack` and'); prn('accepted by `npm install`.'); @@ -184,7 +186,8 @@ exports.build = function build(argv, callback) { // if it is .gitignored (as they should be). So, create an empty .npmignore // if there is a .gitignore but not a .npmignore so build products are // packed. - if (fs.existsSync('.gitignore')) { + if (fs.existsSync('.gitignore') && !fs.existsSync('.npmignore')) { + console.warn('%s: creating an empty .npmignore (please check)', $0); fs.close(fs.openSync('.npmignore', 'a')); } @@ -205,16 +208,8 @@ exports.build = function build(argv, callback) { // Remove dev dependencies, bundle any others, including manually // installed, deps committed into version control, optional, etc. var bundle = lodash.difference(deps, dev); - - // Two names are allowed for this key... use 'bundledDependencies' unless - // package is already using the other name. - var key = info.bundleDependencies ? - 'bundleDependencies' : - 'bundledDependencies'; - - bundle = lodash.uniq(bundle.concat( - info.bundleDependencies || info.bundledDependencies || [] - )); + var key = preferredDependencyPropertyName(info); + bundle = lodash.uniq(bundle.concat(info[key] || [])); // Re-write package.json, preserving its format if possible. json('package.json', function(er, p) { @@ -234,6 +229,24 @@ exports.build = function build(argv, callback) { }); } + // Two names are allowed for this key... npm prefers 'bundleDependencies' to + // 'bundledDependencies' (it will only use former if both present, at least as + // of npm v1.4.3). + function preferredDependencyPropertyName(info) { + var PREFER = 'bundleDependencies'; + var ACCEPT = 'bundledDependencies'; + var use; + if (info[ACCEPT]) use = ACCEPT; + if (info[PREFER]) { + if (use) { + console.warn('%s: ignoring `%s` in favor of `%s`', $0, use, PREFER); + } + use = PREFER; + } + if (!use) use = PREFER; + return use; + } + function doNpmPack(_, callback) { runCommand('npm pack', function(er, output) { if (er) { diff --git a/test/test-build-bundled.js b/test/test-build-bundled.js index 71a1065..a0f0eed 100644 --- a/test/test-build-bundled.js +++ b/test/test-build-bundled.js @@ -12,10 +12,10 @@ require('./build-example')(['--install', '-p', '--bundle'], function(er) { return _ === name; }) >= 0; } - debug('package bundles:', info.bundledDependencies); - assert(isBundled(info.bundledDependencies, 'loopback')); - assert(isBundled(info.bundledDependencies, 'loopback-connector-mysql')); - assert(!isBundled(info.bundledDependencies, 'mocha')); + debug('package bundles:', info.bundleDependencies); + assert(isBundled(info.bundleDependencies, 'loopback')); + assert(isBundled(info.bundleDependencies, 'loopback-connector-mysql')); + assert(!isBundled(info.bundleDependencies, 'mocha')); tar.list(tgz, function(er, paths) { debug('tarfile %s bundles:', tgz, bundled); @@ -30,7 +30,7 @@ require('./build-example')(['--install', '-p', '--bundle'], function(er) { return undefined; }).compact().sort().value(); debug('tarfile %s bundles:', tgz, bundled); - debug('should be:', info.bundledDependencies); - assert.deepEqual(bundled, info.bundledDependencies); + debug('should be:', info.bundleDependencies); + assert.deepEqual(bundled, info.bundleDependencies); }); }); From 2b9b56f2532475b3a111f41db004f65145fddaa6 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Tue, 27 May 2014 19:48:13 -0700 Subject: [PATCH 17/23] bundle: bundle only dependencies and optional deps Checking node_modules to avoid optional deps that were not installed is too unpredictable, and always fails if done without first doing an `npm install`. We now do the predictable thing, bundle deps and optional deps. If this isn't sufficient, the user can provide their own bundledDependencies specification in the package.json. --- index.js | 53 ++++++++++++++++++++--------------------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/index.js b/index.js index f9a40ba..20d90ae 100644 --- a/index.js +++ b/index.js @@ -194,22 +194,27 @@ exports.build = function build(argv, callback) { // node_modules is unconditionally ignored by npm pack, the only way to get // the dependencies packed is to name them in the package.json's // bundledDepenencies. - var deps = fs.readdirSync('node_modules').filter(function(file) { - // Only directories containing a package.json are packages. - return shell.test( - '-f', - path.join('node_modules', file, 'package.json') - ); - }); - var info = require(path.resolve('package.json')); - var dev = Object.keys(info.devDependencies || {}); - // Remove dev dependencies, bundle any others, including manually - // installed, deps committed into version control, optional, etc. - var bundle = lodash.difference(deps, dev); - var key = preferredDependencyPropertyName(info); - bundle = lodash.uniq(bundle.concat(info[key] || [])); + var bundled = info.bundleDependencies || info.bundledDependencies; + + debug('found bundled: %j', bundled); + + if (info.bundleDependencies || info.bundledDependencies) { + // Use package specified dependency bundling + return callback(); + } + + // Bundle non-dev dependencies. Optional deps may fail to build at deploy + // time, that's OK, but must be present during packing. If the user has + // more specific desires, they can configure the dependencies themselves, or + // just not run the --bundle action. + bundled = lodash.union( + Object.keys(info.dependencies), + Object.keys(info.optionalDependencies) + ).sort(); + + debug('saving bundled: %j', bundled); // Re-write package.json, preserving its format if possible. json('package.json', function(er, p) { @@ -218,7 +223,7 @@ exports.build = function build(argv, callback) { return callback(er); } - p.data[key] = bundle; + p.data.bundleDependencies = bundled; p.save(function(er) { if (er) { @@ -229,24 +234,6 @@ exports.build = function build(argv, callback) { }); } - // Two names are allowed for this key... npm prefers 'bundleDependencies' to - // 'bundledDependencies' (it will only use former if both present, at least as - // of npm v1.4.3). - function preferredDependencyPropertyName(info) { - var PREFER = 'bundleDependencies'; - var ACCEPT = 'bundledDependencies'; - var use; - if (info[ACCEPT]) use = ACCEPT; - if (info[PREFER]) { - if (use) { - console.warn('%s: ignoring `%s` in favor of `%s`', $0, use, PREFER); - } - use = PREFER; - } - if (!use) use = PREFER; - return use; - } - function doNpmPack(_, callback) { runCommand('npm pack', function(er, output) { if (er) { From 5ee43429a471666b9c893055c18101eed81b9fd4 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Tue, 27 May 2014 19:09:56 -0700 Subject: [PATCH 18/23] commit: commit build products into git --- index.js | 46 ++++++++++++++++++++++++++++++++++++++++++---- lib/git.js | 30 +++++++++++++++++++++++++++++- test/test-onto.js | 23 +++++++++++++++++++++-- 3 files changed, 92 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index 20d90ae..72b2b9b 100644 --- a/index.js +++ b/index.js @@ -18,7 +18,8 @@ function printHelp($0, prn) { prn(''); prn('When using git, the current branch can be committed onto a deployment'); prn('branch before the build. This allows the build products to be'); - prn('committed and synchronized with a source-only branch.'); + prn('committed and synchronized with a source-only branch. Afterwards, the'); + prn('deployment branch will be checked-out, ready for build and commit.'); prn(''); prn('Packages are built without running scripts (by default, see --scripts)'); prn('to avoid compiling any binary addons. Compilation and install scripts'); @@ -40,6 +41,9 @@ function printHelp($0, prn) { prn('Pack output is a tar file in the format produced by `npm pack` and'); prn('accepted by `npm install`.'); prn(''); + prn('Committing the build output allows it to be pushed to a PaaS, or'); + prn('tracked in git. It is committed to current branch (but see `--onto`).'); + prn(''); prn('Options:'); prn(' -h,--help Print this message and exit.'); prn(' -v,--version Print version and exit.'); @@ -48,6 +52,7 @@ function printHelp($0, prn) { prn(' --scripts If installing, run scripts (to build addons).'); prn(' -b,--bundle Modify package to bundle deployment dependencies.'); prn(' -p,--pack Pack into a publishable archive (with dependencies).'); + prn(' -c,--commit Commit build output to current branch.'); } function runCommand(cmd, callback) { @@ -74,8 +79,17 @@ exports.build = function build(argv, callback) { var $0 = process.env.SLC_COMMAND ? 'slc ' + process.env.SLC_COMMAND : path.basename(argv[1]); - var parser = new Parser( - ':v(version)h(help)s(scripts)i(install)b(bundle)p(pack)O:(onto)', + var parser = new Parser([ + ':v(version)', + 'h(help)', + 's(scripts)', + 'i(install)', + 'b(bundle)', + 'p(pack)', + 'O:(onto)', + 'c(commit)', + 'N(no-commit)', + ].join(''), argv); var option; var onto; @@ -83,6 +97,7 @@ exports.build = function build(argv, callback) { var scripts; var bundle; var pack; + var commit; while ((option = parser.getopt()) !== undefined) { switch (option.option) { @@ -107,6 +122,12 @@ exports.build = function build(argv, callback) { case 'O': onto = option.optarg; break; + case 'c': + commit = true; + break; + case 'N': + commit = false; + break; default: console.error('Invalid usage (near option \'%s\'), try `%s --help`.', option.optopt, $0); @@ -119,7 +140,9 @@ exports.build = function build(argv, callback) { return callback(Error('usage')); } - if (!onto && !install && !bundle && !pack) { + // With no actions selected, do everything we can (onto requires an argument, + // so we can't do it automatically). + if (!onto && !install && !bundle && !pack && !commit) { install = bundle = pack = true; } @@ -141,6 +164,10 @@ exports.build = function build(argv, callback) { steps.push(doNpmPack); } + if (commit) { + steps.push(doGitCommit); + } + vasync.pipeline({funcs: steps}, callback); function doGitOnto(_, callback) { @@ -251,4 +278,15 @@ exports.build = function build(argv, callback) { return callback(); }); } + + function doGitCommit(_, callback) { + try { + var info = git.commitAll(); + console.log('%s: committed build products onto `%s`', $0, info.branch); + return callback(); + } catch(er) { + console.error('%s: %s', $0, er.message); + return callback(er); + } + } }; diff --git a/lib/git.js b/lib/git.js index d6175f1..e5498ef 100644 --- a/lib/git.js +++ b/lib/git.js @@ -26,6 +26,15 @@ function execSync(cmd, error) { return exec.output; } +exports.isGit = function gitIsGit() { + try { + execSync('git symbolic-ref --short HEAD', '(do not care)'); + return true; + } catch(er) { + return false; + } +} + exports.onto = function gitOnto($0, gitDstBranch) { // Get name of current branch, and the tree SHA of its HEAD commit var gitSrcBranch = execSync( @@ -48,7 +57,7 @@ exports.onto = function gitOnto($0, gitDstBranch) { // Commit the source tree to the head of the destination branch, and update // the destination branch ref. - var message = fmt('Build `%s` onto `%s`', gitSrcBranch, gitDstBranch); + var message = fmt('Commit branch `%s` onto `%s`', gitSrcBranch, gitDstBranch); var commitSha = execSync( fmt('git commit-tree -p HEAD -p "%s" -m \'%s\' %s', gitDstBranch, message, gitSrcTreeSha), @@ -73,3 +82,22 @@ exports.onto = function gitOnto($0, gitDstBranch) { commitSha: commitSha, }; } + +exports.commitAll = function gitCommitAll() { + var gitSrcBranch = execSync( + 'git symbolic-ref --short HEAD', + 'failed to parse branch `HEAD`, is this a git repository?' + ); + execSync( + fmt('git add --force --all .'), + fmt('failed to add build products to `%s`', gitSrcBranch) + ); + execSync( + fmt('git commit -m "Commit build products"'), + fmt('failed to commit build products to `%s`', gitSrcBranch) + ); + + return { + branch: gitSrcBranch, + }; +} diff --git a/test/test-onto.js b/test/test-onto.js index 4bbeafd..70a458e 100644 --- a/test/test-onto.js +++ b/test/test-onto.js @@ -4,6 +4,7 @@ var fs = require('fs'); var path = require('path'); var tar = require('tar'); var util = require('util'); +var vasync = require('vasync'); var zlib = require('zlib'); var shell = require('shelljs/global'); @@ -35,9 +36,27 @@ assert(!branchEqual('src', 'dst')); var build = require('../'); -build.build(['node', 'main.js', '--onto=dst'], function(er) { +function onto(_, callback) { + build.build(['node', 'main.js', '--onto=dst'], function(er) { + if (er) return callback(er); + assert(branchEqual('src', 'dst')); + return callback(); + }); +} + +// simulate build, by generating output +touch('build.out'); + +function commit(_, callback) { + build.build(['node', 'main.js', '-c'], function(er) { + if (er) return callback(er); + run('git log build.out'); // check build.out was committed + return callback(); + }); +} + +vasync.pipeline({funcs: [onto, commit]}, function(er) { assert.ifError(er); - assert(branchEqual('src', 'dst')); ok = true; }); From 1e06a37fec1b25063ca2a1961e06b7c4c41fda69 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Tue, 27 May 2014 20:18:13 -0700 Subject: [PATCH 19/23] git: remove unused $0 argument to git.onto() --- index.js | 2 +- lib/git.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 72b2b9b..9cd250f 100644 --- a/index.js +++ b/index.js @@ -172,7 +172,7 @@ exports.build = function build(argv, callback) { function doGitOnto(_, callback) { try { - var info = git.onto($0, onto); + var info = git.onto(onto); console.log('%s: merged `%s` onto `%s`, ready to build', $0, info.srcBranch, info.dstBranch); return callback(); diff --git a/lib/git.js b/lib/git.js index e5498ef..4da4ba0 100644 --- a/lib/git.js +++ b/lib/git.js @@ -35,7 +35,7 @@ exports.isGit = function gitIsGit() { } } -exports.onto = function gitOnto($0, gitDstBranch) { +exports.onto = function gitOnto(gitDstBranch) { // Get name of current branch, and the tree SHA of its HEAD commit var gitSrcBranch = execSync( 'git symbolic-ref --short HEAD', From 31ca4cb6f25fd2abcd949c199ec677e809d36043 Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Tue, 27 May 2014 20:12:19 -0700 Subject: [PATCH 20/23] bundle,pack: log the actions taken --- index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 9cd250f..aee70d0 100644 --- a/index.js +++ b/index.js @@ -255,8 +255,10 @@ exports.build = function build(argv, callback) { p.save(function(er) { if (er) { console.error('%s: error writing package.json: %s', $0, er.message); + return callback(er); } - return callback(er); + console.log('%s: bundled dependencies into package.json', $0); + return callback(); }); }); } @@ -275,6 +277,8 @@ exports.build = function build(argv, callback) { shell.mv('-f', src, dst); + console.log('%s: pack into `%s`', $0, dst); + return callback(); }); } From 0315419be0004fd875019c0edac42b772727eb2d Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Wed, 21 May 2014 20:00:15 -0700 Subject: [PATCH 21/23] fixup! TODO (will rebase away before when complete) --- TODO.txt | 105 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 TODO.txt diff --git a/TODO.txt b/TODO.txt new file mode 100644 index 0000000..178905a --- /dev/null +++ b/TODO.txt @@ -0,0 +1,105 @@ +Options: + +X append git build to another branch +X build and package node binaries, or not +- (do later, could be useful for testing) bundle dev dependencies, or not +- `-v,--verbose`, to print all commands and their output to stdout + +https://github.com/davepacheco/node-vasync + +# GIT + +X merge current branch onto a destination branch before building it + +- (later?) TBD support a --from option to merge from other than current branch, or else + a current state that is NOT a branch (like `git co v2.0.2`, a detached head) + +- (later?) TBD Might need to give a commit message or template for the new commit + +X commit build products + + $ git add --force --all . && git commit + +- remove $0 from lib/git.js + +Build: + +X npm install --ignore-scripts (don't ignore scripts if --binary is requested) +- (no, we don't know the target to run) ... run grunt? +- (could run bower, do later, use build script for now) ... run bower? +X ... npm run build (sl specific) ... maybe make people put "grunt" in as + the build script? maybe just run it if its there? note: don't have + to check return of npm run build, it suceeds if no such script... + + https://www.npmjs.org/doc/misc/npm-scripts.html + + can't use prepublish... because it will be run later on deploy server + during `npm install` :-( + +Package: + +X git add --force --all . && git commit + +- TBD git archive HEAD --prefix=package -o out.tar.gz/out.tar (autodetects format) + +Script the archive, but ensure that the tarballs have same format as +npm packed tarballs. + +Don't do, since we aren't building a development-branch: + +- .gitignore update: npm rebuild; git status|grep new >> .gitignore +- git ci --amend .gitignore + +Problem with npm rebuild... loopback oracle :-( It downloads at run-time, +so demonstrates that a sufficiently weird npm package might still require +network access: + +``` +sam@samtu:~/w/sn/sls-sample-app (try-build) % npm rebuild + +> loopback-oracle-installer@1.2.0 install /home/sam/w/sn/sls-sample-app/node_modules/loopback-connector-oracle/node_modules/loopback-oracle-installer +> node pkginstall.js + +Downloading/Extracting http://7e9918db41dd01dbf98e-ec15952f71452bc0809d79c86f5751b6.r22.cf1.rackcdn.com/loopback-oracle-Linux-x64-1.2.0.tar.gz +``` + +# non-git + +Build: + +(as above) + +Package: + +X create an empty .npmignore if one doesn't exist but .gitignore does, + otherwise npm ignores any build products in .gitignore + +node_modules is unconditionally ignored, see +https://www.npmjs.org/doc/developers.html. So, we will have to bundle the deps. + +X npm dependency bundling + +Note an issue with my approach of looking in node_modules... it means that peer +deps of dev dependencies will get packaged... perhaps I should use only the +deps in the package.json (and MAYBE check for existence in node_modules for +optional packages). recall the npm install on build server will attempt to +install any missing optional packages, so having optional packages for a +deployable app that are missing is pretty suspicous, and not likely to do +anything useful. + +X don't read node_modules to find deps to bundle, require presence + in package.json + +X don't bundle deps if they already have a spec for bundled, otherwise, + bundle all non-dev + +X npm pack + +X put tarball one directory up, instead of CWD? + +- TBD test against a repo with a build script, perhaps grunt or bower + +Tried git@github.com:strongloop/loopback-example-full-stack + +Doesn't work for now... I can't get the app to work manually, I need a +walk-through. From 5a1928ee5650f47fe338980e0a03c508e91a00ec Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Tue, 27 May 2014 21:16:35 -0700 Subject: [PATCH 22/23] readme: update with latest usage --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8f047a1..c22d576 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ With no options, the default is to install, bundle, and pack. When using git, the current branch can be committed onto a deployment branch before the build. This allows the build products to be -committed and synchronized with a source-only branch. +committed and synchronized with a source-only branch. Afterwards, the +deployment branch will be checked-out, ready for build and commit. Packages are built without running scripts (by default, see --scripts) to avoid compiling any binary addons. Compilation and install scripts @@ -23,11 +24,16 @@ be run after installing dependencies. Bundling configures the package.json and .npmignore so deployment (not development) dependencies as well as any 'build' script output will -not be ignored by `npm pack`. +not be ignored by `npm pack`. If an .npmignore is created, check it +carefully to ensure build products are packed, but editor and test +ephemera are not. Pack output is a tar file in the format produced by `npm pack` and accepted by `npm install`. +Committing the build output allows it to be pushed to a PaaS, or +tracked in git. It is committed to current branch (but see `--onto`). + Options: * -h,--help Print this message and exit. * -v,--version Print version and exit. @@ -36,3 +42,4 @@ Options: * --scripts If installing, run scripts (to build addons). * -b,--bundle Modify package to bundle deployment dependencies. * -p,--pack Pack into a publishable archive (with dependencies). +* -c,--commit Commit build output to current branch. From 82093516e83d3816c82095d07917d1613c59b10b Mon Sep 17 00:00:00 2001 From: Sam Roberts Date: Tue, 27 May 2014 21:29:34 -0700 Subject: [PATCH 23/23] log: ensure each action logs on complete --- index.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index aee70d0..83c65d5 100644 --- a/index.js +++ b/index.js @@ -42,7 +42,8 @@ function printHelp($0, prn) { prn('accepted by `npm install`.'); prn(''); prn('Committing the build output allows it to be pushed to a PaaS, or'); - prn('tracked in git. It is committed to current branch (but see `--onto`).'); + prn('tracked in git. It is committed to current branch, and best combined'); + prn('with `--onto` to avoid having build products on non-deploy branches.'); prn(''); prn('Options:'); prn(' -h,--help Print this message and exit.'); @@ -173,7 +174,7 @@ exports.build = function build(argv, callback) { function doGitOnto(_, callback) { try { var info = git.onto(onto); - console.log('%s: merged `%s` onto `%s`, ready to build', + console.log('%s: merged `%s` onto `%s`', $0, info.srcBranch, info.dstBranch); return callback(); } catch(er) { @@ -193,6 +194,7 @@ exports.build = function build(argv, callback) { reportRunError(er, output); return callback(er); } + console.log('%s: installed with `%s`', $0, npmInstall); return doBuildScript(_, callback); }); } @@ -202,8 +204,10 @@ exports.build = function build(argv, callback) { if (er) { console.error('%s: error in package build script', $0); reportRunError(er, output); + return callback(er); } - return callback(er); + console.log('%s: ran custom build with `npm run build`', $0); + return callback(); }); } @@ -257,7 +261,7 @@ exports.build = function build(argv, callback) { console.error('%s: error writing package.json: %s', $0, er.message); return callback(er); } - console.log('%s: bundled dependencies into package.json', $0); + console.log('%s: saved bundled dependencies in package.json', $0); return callback(); }); }); @@ -277,7 +281,7 @@ exports.build = function build(argv, callback) { shell.mv('-f', src, dst); - console.log('%s: pack into `%s`', $0, dst); + console.log('%s: packed into `%s`', $0, dst); return callback(); });