Skip to content

Commit d064e53

Browse files
committed
feat(puppet): switch to chrome
1 parent 0530a35 commit d064e53

39 files changed

+655
-641
lines changed

.github/workflows/lint-and-test.yml

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717

1818
- uses: actions/setup-node@v1
1919
with:
20-
node-version: 12
20+
node-version: 14
2121

2222
- name: Get yarn cache directory path
2323
id: yarn-cache-dir-path
@@ -65,7 +65,7 @@ jobs:
6565
fail-fast: false
6666
matrix:
6767
os: [macos-latest, windows-latest, ubuntu-latest]
68-
node-version: [12, 14]
68+
node-version: [13, 14]
6969

7070
runs-on: ${{ matrix.os }}
7171

@@ -75,10 +75,12 @@ jobs:
7575
- uses: actions/setup-node@v1
7676
with:
7777
node-version: ${{ matrix.node-version }}
78+
7879
- uses: actions/setup-go@v2
7980
with:
8081
go-version: 1.14
81-
- name: Download a single artifact
82+
83+
- name: Download built typescript files
8284
uses: actions/download-artifact@v2
8385
with:
8486
name: js-build
@@ -115,8 +117,18 @@ jobs:
115117
ELECTRON_SKIP_BINARY_DOWNLOAD: 1
116118
SA_REPLAY_SKIP_BINARY_DOWNLOAD: 1
117119

120+
- name: Linux - Apt Install Chrome(s)
121+
if: ${{ matrix.os == 'ubuntu-latest' }}
122+
run: |
123+
set -x
124+
for deb in /tmp/secret-agent/browser-installers/*.deb; do
125+
sudo chown _apt "$deb"
126+
echo "sudo apt -y install $deb"
127+
sudo apt -y install "$deb"
128+
done
129+
118130
- name: Run tests
119-
run: yarn jest --testTimeout=60000 --runInBand --detectOpenHandles
131+
run: yarn jest --testTimeout=60000
120132
working-directory: ./build
121133
env:
122134
SA_SHOW_REPLAY: false
@@ -125,6 +137,6 @@ jobs:
125137
- name: Coverage
126138
run: npm -g install codecov && codecov
127139
working-directory: ./build
128-
if: ${{ matrix.node-version == '12' && matrix.os == 'ubuntu-latest' }}
140+
if: ${{ matrix.node-version == '14' && matrix.os == 'ubuntu-latest' }}
129141
env:
130142
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/new-version.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@ jobs:
1616
- name: Install Node.js, NPM and Yarn
1717
uses: actions/setup-node@v1
1818
with:
19-
node-version: 12
19+
node-version: 14
2020

2121
- name: Yarn install
2222
run: yarn install --network-timeout 1000000
23+
env:
24+
SA_SKIP_DOWNLOAD: true
2325

2426
- name: Build
2527
run: yarn build

commons/downloadFile.ts

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
import * as http from 'http';
2+
import { HttpsProxyAgent } from 'https-proxy-agent';
3+
import * as https from 'https';
4+
import { RequestOptions } from 'https';
5+
import { parse } from 'url';
6+
import { createWriteStream } from 'fs';
7+
import { createPromise } from './utils';
8+
import { getProxyForUrl } from './getProxyForUrl';
9+
10+
export default function downloadFile(
11+
url: string,
12+
destinationPath: string,
13+
progressCallback?: (downloadedBytes: number, totalBytes: number) => void,
14+
): Promise<void> {
15+
const downloaderPromise = createPromise<void>();
16+
let downloadedBytes = 0;
17+
let totalBytes = 0;
18+
19+
const request = httpGet(url, response => {
20+
if (response.statusCode !== 200) {
21+
const error = new Error(
22+
`Download failed: server returned code ${response.statusCode}. URL: ${url}`,
23+
);
24+
// consume response data to free up memory
25+
response.resume();
26+
downloaderPromise.reject(error);
27+
return;
28+
}
29+
const file = createWriteStream(destinationPath);
30+
file.once('finish', downloaderPromise.resolve);
31+
file.once('error', downloaderPromise.reject);
32+
response.pipe(file);
33+
totalBytes = parseInt(response.headers['content-length'], 10);
34+
if (progressCallback) response.on('data', onData);
35+
});
36+
request.once('error', downloaderPromise.reject);
37+
return downloaderPromise.promise;
38+
39+
function onData(chunk: Buffer | string): void {
40+
downloadedBytes += Buffer.byteLength(chunk);
41+
progressCallback(downloadedBytes, totalBytes);
42+
}
43+
}
44+
45+
function httpGet(url: string, response: (x: http.IncomingMessage) => void): http.ClientRequest {
46+
const options = getRequestOptionsWithProxy(url);
47+
const httpModule = options.protocol === 'https:' ? https : http;
48+
49+
const request = httpModule.request(options, (res: http.IncomingMessage): void => {
50+
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
51+
httpGet(res.headers.location, response);
52+
} else {
53+
response(res);
54+
}
55+
});
56+
request.end();
57+
return request;
58+
}
59+
60+
function getRequestOptionsWithProxy(url: string): RequestOptions {
61+
const urlParsed = parse(url);
62+
63+
const options: https.RequestOptions = {
64+
...urlParsed,
65+
method: 'GET',
66+
};
67+
68+
const proxyURL = getProxyForUrl(url);
69+
if (proxyURL) {
70+
if (url.startsWith('http:')) {
71+
return {
72+
path: urlParsed.href,
73+
host: proxyURL.hostname,
74+
port: proxyURL.port,
75+
};
76+
}
77+
options.agent = new HttpsProxyAgent(proxyURL.href);
78+
options.rejectUnauthorized = false;
79+
}
80+
return options;
81+
}

commons/getProxyForUrl.ts

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import { URL } from 'url';
2+
3+
// logic from https://github.com/Rob--W/proxy-from-env
4+
5+
export function getProxyForUrl(url: string): URL {
6+
const parsedUrl = new URL(url);
7+
let protocol = parsedUrl.protocol;
8+
let hostname = parsedUrl.host;
9+
if (typeof hostname !== 'string' || !hostname || typeof protocol !== 'string') {
10+
return null; // Don't proxy URLs without a valid scheme or host.
11+
}
12+
13+
setProxyVars();
14+
protocol = protocol.split(':', 1)[0];
15+
// Stripping ports in this way instead of using parsedUrl.hostname to make
16+
// sure that the brackets around IPv6 addresses are kept.
17+
hostname = hostname.replace(/:\d*$/, '');
18+
let parsedPort = parseInt(parsedUrl.port, 10) || 0;
19+
if (!parsedPort) parsedPort = protocol === 'https:' ? 443 : 80;
20+
if (!shouldProxy(hostname, parsedPort)) {
21+
return null; // Don't proxy URLs that match NO_PROXY.
22+
}
23+
24+
let proxy =
25+
getEnv(`npm_config_${protocol}_proxy`) ||
26+
getEnv(`${protocol}_proxy`) ||
27+
getEnv('npm_config_proxy') ||
28+
getEnv('all_proxy');
29+
if (proxy && proxy.indexOf('://') === -1) {
30+
// Missing scheme in proxy, default to the requested URL's scheme.
31+
proxy = `${protocol}://${proxy}`;
32+
}
33+
if (!proxy) return null;
34+
return new URL(proxy);
35+
}
36+
37+
function shouldProxy(hostname: string, port: number): boolean {
38+
const NO_PROXY = (getEnv('npm_config_no_proxy') || getEnv('no_proxy')).toLowerCase();
39+
if (!NO_PROXY) {
40+
return true; // Always proxy if NO_PROXY is not set.
41+
}
42+
if (NO_PROXY === '*') {
43+
return false; // Never proxy if wildcard is set.
44+
}
45+
46+
for (const proxy of NO_PROXY.split(/[,\s]/)) {
47+
if (!proxy) continue;
48+
const parsedProxy = proxy.match(/^(.+):(\d+)$/);
49+
let parsedProxyHostname = parsedProxy ? parsedProxy[1] : proxy;
50+
const parsedProxyPort = parsedProxy ? parseInt(parsedProxy[2], 10) : 0;
51+
if (parsedProxyPort && parsedProxyPort !== port) {
52+
continue;
53+
}
54+
55+
if (!parsedProxyHostname.startsWith('.') && !parsedProxyHostname.startsWith('*')) {
56+
// No wildcards, so stop proxying if there is an exact match.
57+
if (hostname === parsedProxyHostname) {
58+
return false;
59+
}
60+
}
61+
62+
if (parsedProxyHostname.startsWith('*')) {
63+
// Remove leading wildcard.
64+
parsedProxyHostname = parsedProxyHostname.slice(1);
65+
}
66+
// Stop proxying if the hostname ends with the no_proxy host.
67+
if (!hostname.endsWith(parsedProxyHostname)) {
68+
return false;
69+
}
70+
}
71+
return true;
72+
}
73+
74+
function getEnv(key: string): string {
75+
return process.env[key.toLowerCase()] || process.env[key.toUpperCase()] || '';
76+
}
77+
78+
function setProxyVars(): void {
79+
// Override current environment proxy settings with npm configuration, if any.
80+
const NPM_HTTPS_PROXY = process.env.npm_config_https_proxy || process.env.npm_config_proxy;
81+
const NPM_HTTP_PROXY = process.env.npm_config_http_proxy || process.env.npm_config_proxy;
82+
const NPM_NO_PROXY = process.env.npm_config_no_proxy;
83+
84+
if (NPM_HTTPS_PROXY) process.env.HTTPS_PROXY = NPM_HTTPS_PROXY;
85+
if (NPM_HTTP_PROXY) process.env.HTTP_PROXY = NPM_HTTP_PROXY;
86+
if (NPM_NO_PROXY) process.env.NO_PROXY = NPM_NO_PROXY;
87+
}

commons/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
"@secret-agent/core-interfaces": "1.3.1-alpha.0",
88
"source-map-support": "^0.5.19",
99
"typeson": "^5.18.2",
10-
"typeson-registry": "^1.0.0-alpha.38"
10+
"typeson-registry": "^1.0.0-alpha.38",
11+
"https-proxy-agent": "^5.0.0"
1112
},
1213
"devDependencies": {
1314
"@types/better-sqlite3": "^5.4.1"

core-interfaces/IBrowserEmulatorClass.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import IUserAgentOption from './IUserAgentOption';
44
export default interface IBrowserEmulatorClass {
55
id: string;
66
roundRobinPercent: number;
7-
engine: { browser: string; executablePath: string; revision: string };
7+
engine: { browser: string; executablePath: string; version: string };
88
new (): IBrowserEmulator;
99
userAgentOptions?: IUserAgentOption[];
1010
}

core-interfaces/IBrowserEngine.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
export default interface IBrowserEngine {
22
browser: string;
3-
revision: string;
3+
version: string;
44
executablePath: string;
5+
extraLaunchArgs?: string[];
56
}

core/test/domRecorder.test.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ function sort() {
247247
expect(domFrames).toHaveLength(3);
248248

249249
await tab.puppetPage.frames[3].waitForLoad();
250+
await state.db.flush();
250251
const frames = state.db.frames.all();
251252
expect(frames).toHaveLength(4);
252253
const test1 = frames.find(x => x.name === 'test1');

emulate-browsers/base/index.ts

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
import * as browserPaths from '@secret-agent/puppet/lib/browserPaths';
21
import IBrowserEmulatorClass, {
32
BrowserEmulatorClassDecorator,
43
} from '@secret-agent/core-interfaces/IBrowserEmulatorClass';
54
import IBrowserEmulator from '@secret-agent/core-interfaces/IBrowserEmulator';
5+
import IBrowserEngine from '@secret-agent/core-interfaces/IBrowserEngine';
6+
import { EngineFetcher } from '@secret-agent/puppet/lib/EngineFetcher';
7+
import Log from '@secret-agent/commons/Logger';
68
import modifyHeaders from './lib/modifyHeaders';
79
import DataLoader from './lib/DataLoader';
810
import DomDiffLoader from './lib/DomDiffLoader';
@@ -11,8 +13,24 @@ import parseNavigatorPlugins from './lib/parseNavigatorPlugins';
1113
import DomOverridesBuilder from './lib/DomOverridesBuilder';
1214
import * as DnsOverTlsProviders from './lib/DnsOverTlsProviders';
1315

14-
function getEngineExecutablePath(engine: { browser: string; revision: string }) {
15-
return browserPaths.getExecutablePath(engine.browser, engine.revision);
16+
const { log } = Log(module);
17+
18+
function getEngine(
19+
engine: { browser: string; version: string },
20+
executablePathOverride?: string,
21+
): IBrowserEngine {
22+
const engineFetcher = new EngineFetcher(engine.browser, engine.version).toJSON();
23+
24+
if (executablePathOverride) {
25+
engineFetcher.executablePath = executablePathOverride;
26+
}
27+
28+
log.stats('Browser.getEngine', {
29+
sessionId: null,
30+
engineFetcher,
31+
});
32+
33+
return engineFetcher;
1634
}
1735

1836
export {
@@ -25,6 +43,6 @@ export {
2543
DataLoader,
2644
DomDiffLoader,
2745
getTcpSettingsForOs,
28-
getEngineExecutablePath,
46+
getEngine,
2947
parseNavigatorPlugins,
3048
};

emulate-browsers/base/install.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import EngineInstaller from '@secret-agent/puppet/lib/EngineInstaller';
22

3-
export default function install(engine: { browser: string; revision: string }) {
3+
export default function install(engine: { browser: string; version: string }) {
44
new EngineInstaller(engine)
55
.install()
66
.then(() => {

0 commit comments

Comments
 (0)