Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@
"wasm2c": "1.0.0"
},
"scripts": {
"lint": "eslint src/parseTools.js tools/acorn-optimizer.js"
"lint": "eslint src/parseTools.js tools/acorn-optimizer.js tools/lz4-compress.js"
}
}
159 changes: 31 additions & 128 deletions tools/lz4-compress.js
100644 → 100755
Original file line number Diff line number Diff line change
@@ -1,157 +1,60 @@
#!/usr/bin/env node
// Copyright 2015 The Emscripten Authors. All rights reserved.
// Emscripten is available under two separate licenses, the MIT license and the
// University of Illinois/NCSA Open Source License. Both these licenses can be
// found in the LICENSE file.

// *** Environment setup code ***
var arguments_ = [];
var debug = false;
const fs = require('fs');
const path = require('path');

var ENVIRONMENT_IS_NODE = typeof process === 'object';
var ENVIRONMENT_IS_WEB = typeof window === 'object';
var ENVIRONMENT_IS_WORKER = typeof importScripts === 'function';
var ENVIRONMENT_IS_SHELL = !ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_WORKER;
const arguments_ = process['argv'].slice(2);
const debug = false;

if (ENVIRONMENT_IS_NODE) {
Comment on lines -10 to -15

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are not necessary anymore? It seems we use these variables in other js files..?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah this script is only ever run under node these days.. and its standalone. This was just copyied and pasted from another script but not needed here.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where do we guarantee this only runs on node? I'm not very familiar with this script's usage, but searching lz4-compress in the codebase only turns up tools/file_packager.py, which looks it can run elsewhere too... Also do we not run this on d8 as well?

Anyway, if that's the case, does it give a readable error message when someone tries to run this on other environments?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a completely internal tool just like src/compiler.js and tools/preprocessor.js. These tool may once have worked in non-node environments but today we assume they all run under node only. The code here is completely internal to emscripten and there is exactly one call site.

All of these internal JS scripts require node these days and there is no reason anyone should try to run them under d8 (at least not that I know of).

If a d8 user did try to run this script and saw require() is not a function is should be very obvious that they are trying to run a node script outside of node.

// Expose functionality in the same simple way that the shells work
// Note that we pollute the global namespace here, otherwise we break in node
print = function(x) {
process['stdout'].write(x + '\n');
};
printErr = function(x) {
process['stderr'].write(x + '\n');
};

var nodeFS = require('fs');
var nodePath = require('path');

if (!nodeFS.existsSync) {
nodeFS.existsSync = function(path) {
try {
return !!nodeFS.readFileSync(path);
} catch(e) {
return false;
}
}
}

function find(filename) {
var prefixes = [nodePath.join(__dirname, '..', 'src'), process.cwd()];
for (var i = 0; i < prefixes.length; ++i) {
var combined = nodePath.join(prefixes[i], filename);
if (nodeFS.existsSync(combined)) {
return combined;
}
}
return filename;
}

read = function(filename, binary) {
filename = nodePath['normalize'](filename);
var ret = nodeFS['readFileSync'](filename);
// The path is absolute if the normalized version is the same as the resolved.
if (!ret && filename != nodePath['resolve'](filename)) {
filename = path.join(__dirname, '..', 'src', filename);
ret = nodeFS['readFileSync'](filename);
}
Comment on lines -52 to -56

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this part not necessary anymore?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, this was just boilerplate copied from other scripts. Its not needed there.

if (ret && !binary) ret = ret.toString();
return ret;
};

readBinary = function(filename) { return read(filename, true) };

load = function(f) {
globalEval(read(f));
};

arguments_ = process['argv'].slice(2);

} else if (ENVIRONMENT_IS_SHELL) {
// Polyfill over SpiderMonkey/V8 differences
if (!this['read']) {
this['read'] = function(f) { snarf(f) };
}

if (typeof scriptArgs != 'undefined') {
arguments_ = scriptArgs;
} else if (typeof arguments != 'undefined') {
arguments_ = arguments;
}

} else if (ENVIRONMENT_IS_WEB) {
this['print'] = printErr = function(x) {
console.log(x);
};

this['read'] = function(url) {
var xhr = new XMLHttpRequest();
xhr.open('GET', url, false);
xhr.send(null);
return xhr.responseText;
};

if (this['arguments']) {
arguments_ = arguments;
}
} else if (ENVIRONMENT_IS_WORKER) {
// We can do very little here...

this['load'] = importScripts;

} else {
throw 'Unknown runtime environment. Where are we?';
function print(x) {
process['stdout'].write(x + '\n');
}

function globalEval(x) {
eval.call(null, x);
function printErr(x) {
process['stderr'].write(x + '\n');
}

if (typeof load === 'undefined' && typeof read != 'undefined') {
this['load'] = function(f) {
globalEval(read(f));
};
function read(filename, binary) {
filename = path['normalize'](filename);
let ret = fs['readFileSync'](filename);
if (ret && !binary) ret = ret.toString();
return ret;
}

if (typeof printErr === 'undefined') {
this['printErr'] = function(){};
function readBinary(filename) {
return read(filename, true);
}

if (typeof print === 'undefined') {
this['print'] = printErr;
function globalEval(x) {
eval.call(null, x);
}

assert = function(x, message) {
if (!x) throw 'assertion failed: ' + message + ' : ' + new Error().stack;
function load(f) {
globalEval(read(f));
}

if (!Math['imul'] || Math['imul'](0xffffffff, 5) !== -5) Math['imul'] = function imul(a, b) {
var ah = a >>> 16;
var al = a & 0xffff;
var bh = b >>> 16;
var bl = b & 0xffff;
return (al*bl + ((ah*bl + al*bh) << 16))|0;
assert = function(x, message) {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Other functions in this file have function assert(x, message) { ... form.. Is there a reason why this function is different?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think its because this function needs to be added to the global namespace so that the script that is eval 'ed below had visibility of it.

Its a strange quirk of node and if you just say foo = 3 that creates a new name foo on the global object that is shared.

I have followup PR that will remove the use of eval completely .. but for now this smaller cleanup still relies on eval and the global shared namespace.

if (!x) throw new Errror(message);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like one too many rs in this Error?

};

// Redirect console.log message from MiniLZ4 to stderr since stdout is
// where we return the decompressed data.
console.log = printErr;

// *** Environment setup code ***

var lz4 = arguments_[0];
var input = arguments_[1];
var output = arguments_[2];
const lz4 = arguments_[0];
const input = arguments_[1];
const output = arguments_[2];

load(lz4);

var data = readBinary(input);
if (!(data instanceof ArrayBuffer)) {
printErr('converting to ArrayBuffer');
data = new Uint8Array(data).buffer;
}

var start = Date.now();
var compressedData = MiniLZ4.compressPackage(data);
nodeFS['writeFileSync'](output, Buffer.from(compressedData['data']));
const data = new Uint8Array(readBinary(input)).buffer;
const start = Date.now();
const compressedData = MiniLZ4.compressPackage(data);
fs.writeFileSync(output, Buffer.from(compressedData['data']));
compressedData['data'] = null;
printErr('compressed in ' + (Date.now() - start) + ' ms');
print(JSON.stringify(compressedData));