From 76c9e7deb9cc97bf44dadf7bc85eed94312ebb01 Mon Sep 17 00:00:00 2001 From: Sean Monahan Date: Wed, 24 Aug 2022 17:24:52 -0700 Subject: [PATCH 1/5] stress-test: add cli application Adds a CLI application for Stress Test that allows test configurations to easily be generated and run. This makes running large test suites as simple as a single command and eliminates the need for most npm scripts that previously existed for this package. --- apps/stress-test/.gitignore | 4 +- apps/stress-test/README.md | 42 +++++- apps/stress-test/benchmarks/browsers.js | 12 -- .../benchmarks/simple-stress/buildConfig.js | 82 ------------ .../simple-stress/processResults.js | 34 ----- apps/stress-test/package.json | 22 +--- apps/stress-test/scenarios/default.js | 55 ++++++++ .../scripts/commands/buildTestConfig.js | 124 ++++++++++++++++++ .../scripts/commands/processResults.js | 56 ++++++++ apps/stress-test/scripts/commands/run.js | 56 ++++++++ .../stress-test/scripts/commands/runServer.js | 24 ++++ .../scripts/commands/runTachometer.js | 54 ++++++++ apps/stress-test/scripts/stressTest.js | 12 ++ .../scripts/utils/configureYargs.js | 119 +++++++++++++++++ .../scripts/utils/getScenarioConfig.js | 23 ++++ apps/stress-test/scripts/utils/paths.js | 79 +++++++++++ .../scripts/utils/processOptions.js | 23 ++++ apps/stress-test/scripts/utils/server.js | 61 +++++++++ apps/stress-test/scripts/utils/tachometer.js | 31 +++++ 19 files changed, 758 insertions(+), 155 deletions(-) delete mode 100644 apps/stress-test/benchmarks/browsers.js delete mode 100644 apps/stress-test/benchmarks/simple-stress/buildConfig.js delete mode 100644 apps/stress-test/benchmarks/simple-stress/processResults.js create mode 100644 apps/stress-test/scenarios/default.js create mode 100644 apps/stress-test/scripts/commands/buildTestConfig.js create mode 100644 apps/stress-test/scripts/commands/processResults.js create mode 100644 apps/stress-test/scripts/commands/run.js create mode 100644 apps/stress-test/scripts/commands/runServer.js create mode 100644 apps/stress-test/scripts/commands/runTachometer.js create mode 100644 apps/stress-test/scripts/stressTest.js create mode 100644 apps/stress-test/scripts/utils/configureYargs.js create mode 100644 apps/stress-test/scripts/utils/getScenarioConfig.js create mode 100644 apps/stress-test/scripts/utils/paths.js create mode 100644 apps/stress-test/scripts/utils/processOptions.js create mode 100644 apps/stress-test/scripts/utils/server.js create mode 100644 apps/stress-test/scripts/utils/tachometer.js diff --git a/apps/stress-test/.gitignore b/apps/stress-test/.gitignore index a8dce92731d372..2b77931853383a 100644 --- a/apps/stress-test/.gitignore +++ b/apps/stress-test/.gitignore @@ -2,6 +2,6 @@ node_modules/ dist/ benchmarks/**/*.json -benchmarks/**/results.js +benchmarks/**/*.js -yarn-error.log \ No newline at end of file +yarn-error.log diff --git a/apps/stress-test/README.md b/apps/stress-test/README.md index bdd123caa3a3a4..6c707cbabc6857 100644 --- a/apps/stress-test/README.md +++ b/apps/stress-test/README.md @@ -6,6 +6,12 @@ This application is configured to support Fluent UI v8, v9 and Web Components. Stress Test is a simple Webpack application as Webpack allows us to support Fluent v8, v9 and Web Components in a single app while using each version of Fluent natively (that is, the React versions of Fluent are typical React apps, the Web Component version is a typical web component app). Additionally, Griffel, a dependency of Fluent v9 provides some Webpack loaders so Stress Test needs to be able to use those. +## Glossary + +- **scenario**: A testing scenario for a specific line of investigation. For example, if you wanted to compare the performance of different `Button` implementations you might create a "button-test" scenario for various targets. +- **targets**: Different implementation targets. For example: "v8" for Fluent UI v8, "v9" for Fluent UI v9, "wc" for Fluent UI Web Components. +- **test cases**: Different test cases to run against a given scenario. For example, you might want to test mounting performance for a scenario. + ## Development Use `yarn start` and select `@fluentui/stress-test` from the list of options to start the application in development mode. @@ -20,18 +26,44 @@ The `components` folder is also split by supported Fluent versions and is where The `benchmarks` folder houses Tachometer configurations and test results; and helper scripts for generating configurations and processing results. +The `scripts` folder house the Stress Test CLI app that is used to run tests. + ### Adding test cases Currently, new test cases must be manually added. Add tests cases to the `pages` array in `webpack.config.js` and they will be generated by Webpack. This configuration keeps the different Fluent versions in separate Webpack chunks. The dev server needs to be restarted after changes to the config file. ## Production -Run `yarn workspace @fluentui/stress-test build` from the repo root to build a static production bundle of the application. +**NOTE**: Tests should be run against production builds. While tests can be run against development builds, and this is useful for gathering quick results, the performance characteristics of development and production builds can differ quite a bit. + +Building for production creates a static production application that can be served with a simple HTTP server. + +Run `yarn workspace @fluentui/stress-test build:app` to build only this application. If you do not need to build dependencies like `@fluentui/react` this is much faster. + +Run `yarn workspace @fluentui/stress-test build:local` to build all the dependencies for this application (e.g., `@fluentui/react`, `@fluentui/web-components`) along with the application. This is much slower but useful if this it the first time you're building the app or you've pulled in lots of changes. + +## Using the CLI + +The CLI is used to run tests and process the results of tests. The CLI handles creating configuration files to run tests, spins up an HTTP server and runs the actual tests. Each step can be handled individiually or you can use a convenience command to run everything for a test case. + +Invoke the CLI with + + $ yarn workspace @fluentui/stress-test stress-test + +This will show the help documentation which will provide the details for each command. What follows is a high level overview of the CLI's features. + +### Commands -Run `yarn workspace @fluentui/stress-test serve` to serve the static production bundle. +- `run`: Runs a test end-to-end. Creates configuration files, starts up an HTTP server, runs the tests, and processes the results. +- `build-test-config`: Generates test configuration files only. +- `process-results`: Processes the results of a test run. +- `serve`: Starts the HTTP server. +- `tachometer`: Runs Tachometers against an HTTP server based. -## Running test cases +### Examples -Run `yarn workspace @fluentui/stress-test bench:*` where "\*" is one of the test cases in `package.json`. + # Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against Firefox with small sizes and low sample size + $ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --sample-size 2 --test-cases mount prop-update --browsers firefox --sizes xs s -NOTE: Stress Test must be running, preferrably as a production build. + # Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against the default browsers at the default sizes and sample size + $ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --test-cases mount prop-update diff --git a/apps/stress-test/benchmarks/browsers.js b/apps/stress-test/benchmarks/browsers.js deleted file mode 100644 index ee1d798af2e2a2..00000000000000 --- a/apps/stress-test/benchmarks/browsers.js +++ /dev/null @@ -1,12 +0,0 @@ -// const os = require('os'); - -module.exports.getBrowsers = () => { - // https://github.com/Polymer/tachometer#webdriver-plugins - const browsers = ['chrome', 'firefox' /*'edge'*/]; - - // if (os.type() === 'Darwin') { - // browsers.push('safari'); - // } - - return browsers; -}; diff --git a/apps/stress-test/benchmarks/simple-stress/buildConfig.js b/apps/stress-test/benchmarks/simple-stress/buildConfig.js deleted file mode 100644 index c701e4f64f36dc..00000000000000 --- a/apps/stress-test/benchmarks/simple-stress/buildConfig.js +++ /dev/null @@ -1,82 +0,0 @@ -const fs = require('fs-extra'); -const { getBrowsers } = require('../browsers'); -const path = require('path'); -const yargs = require('yargs'); - -const options = yargs - .options({ - size: { - alias: 's', - describe: 'Number of components to use in the test', - default: 100, - }, - 'add-nodes': { - alias: 'a', - describe: 'Number of components to add in the test', - }, - 'remove-nodes': { - alias: 'r', - describe: 'Number of components to remove in the test', - }, - }) - .parse(); - -if (!options.addNodes) { - options.addNodes = options.size; -} - -if (!options.removeNodes) { - options.removeNodes = options.size - 1; -} - -const makeConfigJson = (browser, testCase, sampleSize = 25) => { - const json = ` - { - "$schema": "https://raw.githubusercontent.com/Polymer/tachometer/master/config.schema.json", - "sampleSize": ${sampleSize}, - "timeout": 0, - "benchmarks": [ - { - "browser": { - "name": "${browser}" - }, - "measurement": [ - { - "mode": "performance", - "entryName": "stress" - } - ], - "expand": [ - { - "name": "v8 - ${testCase}", - "url": "http://localhost:8080/v8/simple-stress/?test=${testCase}&numStartNodes=${options.size}&numAddNodes=${options.addNodes}&numRemoveNodes=${options.removeNodes}" - }, - { - "name": "v9 - ${testCase}", - "url": "http://localhost:8080/v9/simple-stress/?test=${testCase}&numStartNodes=${options.size}&numAddNodes=${options.addNodes}&numRemoveNodes=${options.removeNodes}" - }, - { - "name": "wc - ${testCase}", - "url": "http://localhost:8080/wc/simple-stress/?test=${testCase}&numStartNodes=${options.size}&numAddNodes=${options.addNodes}&numRemoveNodes=${options.removeNodes}" - } - ] - } - ] - } - `; - - return json; -}; - -const browsers = getBrowsers(); -const testCases = ['mount', 'inject-styles', 'prop-update', 'add-node', 'remove-node']; - -fs.mkdirpSync(path.join(__dirname, 'config')); -fs.mkdirpSync(path.join(__dirname, 'results')); - -for (const browser of browsers) { - for (const testCase of testCases) { - const json = makeConfigJson(browser, testCase); - fs.writeFileSync(path.join(__dirname, 'config', [browser, testCase, 'json'].join('.')), json, { encoding: 'utf8' }); - } -} diff --git a/apps/stress-test/benchmarks/simple-stress/processResults.js b/apps/stress-test/benchmarks/simple-stress/processResults.js deleted file mode 100644 index 85a71025fb5a04..00000000000000 --- a/apps/stress-test/benchmarks/simple-stress/processResults.js +++ /dev/null @@ -1,34 +0,0 @@ -const fs = require('fs-extra'); -const { getBrowsers } = require('../browsers'); -const path = require('path'); - -const browsers = getBrowsers(); -const testCases = ['mount', 'inject-styles', 'prop-update', 'add-node', 'remove-node']; - -const browserData = {}; -for (const browser of browsers) { - browserData[browser] = {}; - for (const testCase of testCases) { - const contents = fs.readFileSync(path.join(__dirname, 'results', `${browser}.${testCase}.results.json`), { - encoding: 'utf8', - }); - const json = JSON.parse(contents); - - browserData[browser][testCase] = json.benchmarks.map(test => { - return { - name: test.name, - mean: test.mean, - differences: test.differences, - samples: test.samples, - }; - }); - } -} - -console.log(browserData); - -const js = ` - export const data = ${JSON.stringify(browserData, null, 4)}; -`; - -fs.writeFileSync(path.join(__dirname, 'results', 'results.js'), js, { encoding: 'utf8' }); diff --git a/apps/stress-test/package.json b/apps/stress-test/package.json index 8971bed57c93ce..4f294df751cb26 100644 --- a/apps/stress-test/package.json +++ b/apps/stress-test/package.json @@ -5,28 +5,10 @@ "main": "index.js", "license": "MIT", "scripts": { + "stress-test": "node ./scripts/stressTest.js", "start": "webpack-dev-server --mode=development", "build:local": "lage build --to @fluentui/react @fluentui/web-components --verbose && yarn build:app", - "build:app": "webpack --mode=production", - "serve": "http-server ./dist", - "bench:simple-stress:build": "node ./benchmarks/simple-stress/buildConfig.js", - "bench:simple-stress": "yarn bench:simple-stress:mount && yarn bench:simple-stress:inject-styles && yarn bench:simple-stress:prop-update && yarn bench:simple-stress:add-node && yarn bench:simple-stress:remove-node", - "bench:simple-stress:process": "node ./benchmarks/simple-stress/processResults.js", - "bench:simple-stress:mount": "yarn bench:simple-stress:mount:chrome && yarn bench:simple-stress:mount:firefox", - "bench:simple-stress:inject-styles": "yarn bench:simple-stress:inject-styles:chrome && yarn bench:simple-stress:inject-styles:firefox", - "bench:simple-stress:prop-update": "yarn bench:simple-stress:prop-update:chrome && yarn bench:simple-stress:prop-update:firefox", - "bench:simple-stress:add-node": "yarn bench:simple-stress:add-node:chrome && yarn bench:simple-stress:add-node:firefox", - "bench:simple-stress:remove-node": "yarn bench:simple-stress:remove-node:chrome && yarn bench:simple-stress:remove-node:firefox", - "bench:simple-stress:mount:chrome": "tach --config ./benchmarks/simple-stress/config/chrome.mount.json --json-file ./benchmarks/simple-stress/results/chrome.mount.results.json", - "bench:simple-stress:mount:firefox": "tach --config ./benchmarks/simple-stress/config/firefox.mount.json --json-file ./benchmarks/simple-stress/results/firefox.mount.results.json", - "bench:simple-stress:inject-styles:chrome": "tach --config ./benchmarks/simple-stress/config/chrome.inject-styles.json --json-file ./benchmarks/simple-stress/results/chrome.inject-styles.results.json", - "bench:simple-stress:inject-styles:firefox": "tach --config ./benchmarks/simple-stress/config/firefox.inject-styles.json --json-file ./benchmarks/simple-stress/results/firefox.inject-styles.results.json", - "bench:simple-stress:prop-update:chrome": "tach --config ./benchmarks/simple-stress/config/chrome.prop-update.json --json-file ./benchmarks/simple-stress/results/chrome.prop-update.results.json", - "bench:simple-stress:prop-update:firefox": "tach --config ./benchmarks/simple-stress/config/firefox.prop-update.json --json-file ./benchmarks/simple-stress/results/firefox.prop-update.results.json", - "bench:simple-stress:add-node:chrome": "tach --config ./benchmarks/simple-stress/config/chrome.add-node.json --json-file ./benchmarks/simple-stress/results/chrome.add-node.results.json", - "bench:simple-stress:add-node:firefox": "tach --config ./benchmarks/simple-stress/config/firefox.add-node.json --json-file ./benchmarks/simple-stress/results/firefox.add-node.results.json", - "bench:simple-stress:remove-node:chrome": "tach --config ./benchmarks/simple-stress/config/chrome.remove-node.json --json-file ./benchmarks/simple-stress/results/chrome.remove-node.results.json", - "bench:simple-stress:remove-node:firefox": "tach --config ./benchmarks/simple-stress/config/firefox.remove-node.json --json-file ./benchmarks/simple-stress/results/firefox.remove-node.results.json" + "build:app": "webpack --mode=production" }, "dependencies": { "@fluentui/react": "^8.91.1", diff --git a/apps/stress-test/scenarios/default.js b/apps/stress-test/scenarios/default.js new file mode 100644 index 00000000000000..d2ac6503704b34 --- /dev/null +++ b/apps/stress-test/scenarios/default.js @@ -0,0 +1,55 @@ +/** + * @typedef {Object} TestOptions + * @property {number} numStartNodes + * @property {number} numAddNodes + * @property {number} numRemoveNodes + */ + +/** + * @typedef {Object.} SizeMap + */ + +/** + * @typedef {Object} ScenarioConfig + * @property {string[]} [testCases] + * @property {SizeMap} [sizes] + * @property {string[]} [browsers] + * @property {number} [sampleSize] + * @property {string[]} [targets] + */ + +module.exports = { + testCases: ['mount', 'inject-styles', 'prop-update', 'remove-node', 'add-node'], + + sizes: { + xs: { + numStartNodes: 100, + numAddNodes: 100, + numRemoveNodes: 99, + }, + + s: { + numStartNodes: 250, + numAddNodes: 250, + numRemoveNodes: 249, + }, + + m: { + numStartNodes: 500, + numAddNodes: 500, + numRemoveNodes: 499, + }, + + l: { + numStartNodes: 750, + numAddNodes: 750, + numRemoveNodes: 749, + }, + + xl: { + numStartNodes: 1000, + numAddNodes: 1000, + numRemoveNodes: 999, + }, + }, +}; diff --git a/apps/stress-test/scripts/commands/buildTestConfig.js b/apps/stress-test/scripts/commands/buildTestConfig.js new file mode 100644 index 00000000000000..f68da1a8b26d54 --- /dev/null +++ b/apps/stress-test/scripts/commands/buildTestConfig.js @@ -0,0 +1,124 @@ +const fs = require('fs-extra'); +const path = require('path'); +const { getConfigDir, getResultsDir, ensureClean } = require('../utils/paths'); +const configureYargs = require('../utils/configureYargs'); +const getScenarioConfig = require('../utils/getScenarioConfig'); +const processOptions = require('../utils/processOptions'); +const querystring = require('querystring'); + +/** + * @typedef {Object} CLIBuildTestConfigOptions + * @property {string} scenario + * @property {string[]} testCases + * @property {string[]} sizes + * @property {string[]} browsers + * @property {number} sampleSize + * @property {string[]} targets + * @property {number} port + */ + +/** + * @typedef {Object} ConfigResult + * @property {string} testFile - Path to test configuration file. + * @property {string} resultsFile - Path to where test results should be written. + */ + +const command = 'build-test-config'; +exports.command = command; +exports.describe = 'Builds test configuration files.'; + +exports.builder = yargs => { + configureYargs(command, yargs); +}; + +/** + * @function buildTestConfig + * @param {CLIBuildTestConfigOptions} options + * @returns {ConfigResult[]} Paths to generated config files + */ +const buildTestConfig = options => { + const { scenario, browsers, testCases, sampleSize, targets, sizes, port } = options; + const config = getScenarioConfig(scenario); + const configDir = getConfigDir(scenario); + ensureClean(configDir); + + const configs = []; + + for (const browser of browsers) { + for (const testCase of testCases) { + for (const size of sizes) { + const json = makeConfigJson(scenario, browser, testCase, sampleSize, targets, size, config.sizes[size], port); + const configName = [browser, testCase, size].join('.') + '.json'; + const configPath = path.join(configDir, configName); + fs.writeFileSync(configPath, json, { encoding: 'utf8' }); + + const resultsDir = getResultsDir(scenario); + const configResult = { + configDir, + resultsDir, + testFile: configPath, + resultsFile: path.join(resultsDir, configName), + }; + + configs.push(configResult); + } + } + } + + return configs; +}; + +/** + * @function makeConfigJson + * @param {string} scenario + * @param {string} browser + * @param {string} testCase + * @param {number} sampleSize + * @param {string[]} targets + * @param {string[]} size + * @param {TestOptions} testOptions + * @returns {string} Stringified JSON + */ +const makeConfigJson = (scenario, browser, testCase, sampleSize, targets, size, testOptions, port) => { + const baseUrl = `http://localhost:${port}`; + const json = { + $schema: 'https://raw.githubusercontent.com/Polymer/tachometer/master/config.schema.json', + sampleSize, + timeout: 0, + benchmarks: [ + { + browser: { + name: browser, + }, + measurement: [ + { + mode: 'performance', + entryName: 'stress', + }, + ], + + expand: targets.map(target => { + const params = querystring.stringify({ test: testCase, ...testOptions }); + + return { + name: `${target} - ${testCase} - ${size}`, + url: `${baseUrl}/${target}/${scenario}/?${params}`, + }; + }), + }, + ], + }; + + return JSON.stringify(json, null, 4); +}; + +/** + * + * @param {CLIBuildTestConfigOptions} argv + */ +exports.handler = argv => { + const options = processOptions(argv); + buildTestConfig(options); +}; + +exports.buildTestConfig = buildTestConfig; diff --git a/apps/stress-test/scripts/commands/processResults.js b/apps/stress-test/scripts/commands/processResults.js new file mode 100644 index 00000000000000..15910a5c7d9587 --- /dev/null +++ b/apps/stress-test/scripts/commands/processResults.js @@ -0,0 +1,56 @@ +const fs = require('fs-extra'); +const path = require('path'); +const configureYargs = require('../utils/configureYargs'); +const { getResultsDir, readDirJson } = require('../utils/paths'); + +/** + * @typedef {Object} CLIProcessResultsOptions + * @property {string} scenario + */ + +const command = 'process-results'; +exports.command = command; +exports.describe = 'Processes test results for display with charts and graphs.'; + +exports.builder = yargs => { + configureYargs(command, yargs); +}; + +/** + * + * @param {CLIProcessResultsOptions} argv + */ +exports.handler = argv => { + const { scenario } = argv; + + const resultsDir = getResultsDir(scenario); + const files = readDirJson(resultsDir); + + const browserData = {}; + for (const file of files) { + const contents = fs.readFileSync(path.join(resultsDir, file), { + encoding: 'utf8', + }); + const json = JSON.parse(contents); + const benchmark = json.benchmarks[0]; + const browser = benchmark.browser.name; + const testCase = benchmark.name.split('-')[1].trim(); + + browserData[browser] = browserData[browser] || {}; + + browserData[browser][testCase] = json.benchmarks.map(test => { + return { + name: test.name, + mean: test.mean, + differences: test.differences, + samples: test.samples, + }; + }); + } + + const js = ` + export const data = ${JSON.stringify(browserData, null, 4)}; + `; + + fs.writeFileSync(path.join(resultsDir, 'processed-results.js'), js, { encoding: 'utf8' }); +}; diff --git a/apps/stress-test/scripts/commands/run.js b/apps/stress-test/scripts/commands/run.js new file mode 100644 index 00000000000000..6b4321e3e4f885 --- /dev/null +++ b/apps/stress-test/scripts/commands/run.js @@ -0,0 +1,56 @@ +const processOptions = require('../utils/processOptions'); +const configureYargs = require('../utils/configureYargs'); +const { startServer, stopServer } = require('../utils/server'); +const runTachometer = require('../utils/tachometer'); +const { handler: processResults } = require('./processResults'); +const { buildTestConfig } = require('./buildTestConfig'); + +/** + * @typedef {Object} CLIRunOptions + * @property {string} scenario + * @property {string[]} testCases + * @property {string[]} sizes + * @property {string[]} browsers + * @property {number} sampleSize + * @property {string[]} targets + * @property {boolean} useConfig + * @property {boolean} processResults + * @property {number} port + * @property {string} root + */ + +const command = 'run'; +exports.command = command; +exports.describe = 'Builds configs and runs stress testing.'; + +exports.builder = yargs => { + configureYargs(command, yargs); +}; + +/** + * + * @param {ConfigResult[]} testConfigs + * @param {CLIServerOptions} options + */ +const run = async (testConfigs, options) => { + const { port, root } = options; + await startServer({ port, root }); + await runTachometer(testConfigs); +}; + +/** + * + * @param {CLIRunOptions} argv + */ +exports.handler = argv => { + const options = processOptions(argv); + + const testConfigs = buildTestConfig(options); + run(testConfigs, options).finally(() => { + stopServer(); + + if (options.processResults) { + processResults(options); + } + }); +}; diff --git a/apps/stress-test/scripts/commands/runServer.js b/apps/stress-test/scripts/commands/runServer.js new file mode 100644 index 00000000000000..9535cb24997fb3 --- /dev/null +++ b/apps/stress-test/scripts/commands/runServer.js @@ -0,0 +1,24 @@ +const configureYargs = require('../utils/configureYargs'); +const { startServer } = require('../utils/server'); + +/** + * @typedef {Object} CLIServerOptions + * @property {number} port + * @property {string} root + */ + +const command = 'serve'; +exports.command = command; +exports.describe = 'Runs a test HTTP server.'; + +exports.builder = yargs => { + configureYargs(command, yargs); +}; + +/** + * + * @param {CLIServerOptions} argv + */ +exports.handler = argv => { + startServer(argv); +}; diff --git a/apps/stress-test/scripts/commands/runTachometer.js b/apps/stress-test/scripts/commands/runTachometer.js new file mode 100644 index 00000000000000..5aec7153f8d388 --- /dev/null +++ b/apps/stress-test/scripts/commands/runTachometer.js @@ -0,0 +1,54 @@ +const fs = require('fs-extra'); +const path = require('path'); + +const configureYargs = require('../utils/configureYargs'); +const { getConfigDir, getResultsDir, readDirJson } = require('../utils/paths'); +const runTachometer = require('../utils/tachometer'); + +/** + * @typedef {Object} CLITachometerOptions + * @property {string} scenario + */ + +const command = 'tachometer'; +exports.command = command; +exports.describe = 'Runs Tachometer for a provided scenario.'; + +exports.builder = yargs => { + configureYargs(command, yargs); +}; + +const run = async testConfigs => { + await runTachometer(testConfigs); +}; + +/** + * + * @param {CLITachometerOptions} argv + */ +exports.handler = argv => { + const { scenario } = argv; + + const configDir = getConfigDir(scenario); + const resultsDir = getResultsDir(scenario); + + if (!fs.existsSync(configDir)) { + throw new Error(`"${configDir} does not exist.`); + } + + const configs = []; + const files = readDirJson(configDir); + + for (const file of files) { + const configResult = { + configDir, + resultsDir, + testFile: path.join(configDir, file), + resultsFile: path.join(resultsDir, file), + }; + + configs.push(configResult); + } + + run(configs).then(() => console.log('Tachometer run complete!')); +}; diff --git a/apps/stress-test/scripts/stressTest.js b/apps/stress-test/scripts/stressTest.js new file mode 100644 index 00000000000000..2dbdbd4f231c6c --- /dev/null +++ b/apps/stress-test/scripts/stressTest.js @@ -0,0 +1,12 @@ +const yargs = require('yargs'); + +yargs + .usage('Usage: $0 [options]') + .command(require('./commands/run.js')) + .command(require('./commands/buildTestConfig')) + .command(require('./commands/processResults.js')) + .command(require('./commands/runServer.js')) + .command(require('./commands/runTachometer.js')) + .demandCommand() + .help() + .parse(); diff --git a/apps/stress-test/scripts/utils/configureYargs.js b/apps/stress-test/scripts/utils/configureYargs.js new file mode 100644 index 00000000000000..85f91d1cecfe5e --- /dev/null +++ b/apps/stress-test/scripts/utils/configureYargs.js @@ -0,0 +1,119 @@ +const { getBrowsers } = require('../../dist/getBrowsers'); + +/** + * @typedef {Object. { + let y = yargs.options(options); + + Object.keys(options).forEach(option => { + switch (option) { + case 'test-cases': + case 'browsers': + case 'targets': + y = y.array(option); + break; + + case 'sizes': + y = y.array(option).choices(option, ['xs', 's', 'm', 'l', 'xl']); + break; + + case 'sample-size': + case 'port': + y = y.number(option); + break; + + case 'use-config': + case 'process-results': + y = y.boolean(option); + } + }); +}; + +/** + * + * @param {string} command + * @param {import('yargs').Argv} yargs + */ +const configureYargs = (command, yargs) => { + switch (command) { + case 'build-test-config': { + const { 'process-results': processResults, ...buildTestOptions } = cliOptions; + configure(yargs, buildTestOptions); + break; + } + + case 'run': { + configure(yargs, cliOptions); + break; + } + + case 'process-results': { + const { scenario } = cliOptions; + configure(yargs, { scenario }); + break; + } + + case 'serve': { + const { port, root } = cliOptions; + configure(yargs, { port, root }); + break; + } + + case 'tachometer': { + const { scenario } = cliOptions; + configure(yargs, { scenario }); + break; + } + } +}; + +module.exports = configureYargs; diff --git a/apps/stress-test/scripts/utils/getScenarioConfig.js b/apps/stress-test/scripts/utils/getScenarioConfig.js new file mode 100644 index 00000000000000..7b34faf839aebe --- /dev/null +++ b/apps/stress-test/scripts/utils/getScenarioConfig.js @@ -0,0 +1,23 @@ +const fs = require('fs-extra'); +const path = require('path'); +const { getScenariosDir } = require('./paths'); + +/** + * @function getScenarioConfig + * @param {string} scenarioName + * @returns {ScenarioConfig} + */ +const getScenarioConfig = scenarioName => { + let scenarioConfig = require('../../scenarios/default.js'); + if (fs.existsSync(path.join(getScenariosDir(), `${scenarioName}.js`))) { + const config = require(`../../scenarios/${scenarioName}.js`); + scenarioConfig = { + ...scenarioConfig, + ...config, + }; + } + + return scenarioConfig; +}; + +module.exports = getScenarioConfig; diff --git a/apps/stress-test/scripts/utils/paths.js b/apps/stress-test/scripts/utils/paths.js new file mode 100644 index 00000000000000..7e4c06b6fa1bfd --- /dev/null +++ b/apps/stress-test/scripts/utils/paths.js @@ -0,0 +1,79 @@ +const fs = require('fs-extra'); +const { join } = require('path'); + +/** + * @returns {string} + */ +const getPackageRoot = () => { + return process.cwd(); +}; + +/** + * @param {string} scenario + * @returns {string} + */ +const getConfigDir = scenario => { + return join(getPackageRoot(), 'benchmarks', scenario, 'config'); +}; + +/** + * @param {string} scenario + * @returns {string} + */ +const getResultsDir = scenario => { + return join(getPackageRoot(), 'benchmarks', scenario, 'results'); +}; + +/** + * @returns {string} + */ +const getScenariosDir = () => { + return join(getPackageRoot(), 'scenarios'); +}; + +/** + * @param {string} dir + * @returns {string[]} + */ +const readDirJson = dir => { + return fs.readdirSync(dir).filter(file => file.endsWith('.json')); +}; + +/** + * @param {string} path + * @returns {boolean} + */ +const remove = path => { + if (fs.pathExistsSync(path)) { + fs.removeSync(path); + return true; + } + + return false; +}; + +/** + * @param {string} path + */ +const mkdirp = path => { + if (!fs.pathExistsSync(path)) { + fs.mkdirpSync(path); + } +}; + +/** + * @param {string} path + */ +const ensureClean = path => { + remove(path); + mkdirp(path); +}; + +module.exports = { + getConfigDir, + getResultsDir, + getScenariosDir, + ensureClean, + getPackageRoot, + readDirJson, +}; diff --git a/apps/stress-test/scripts/utils/processOptions.js b/apps/stress-test/scripts/utils/processOptions.js new file mode 100644 index 00000000000000..61d974562d980a --- /dev/null +++ b/apps/stress-test/scripts/utils/processOptions.js @@ -0,0 +1,23 @@ +const getScenarioConfig = require('./getScenarioConfig'); + +/** + * @param {import('yargs').Options} parsedOptions + * @returns {import('yargs').Options} + */ +const processOptions = parsedOptions => { + if (parsedOptions.useConfig) { + const scenarioConfig = getScenarioConfig(parsedOptions.scenario); + + const options = { + ...scenarioConfig, + ...parsedOptions, + sizes: !parsedOptions.sizes && scenarioConfig.sizes ? Object.keys(scenarioConfig.sizes) : parsedOptions.sizes, + }; + + return options; + } + + return parsedOptions; +}; + +module.exports = processOptions; diff --git a/apps/stress-test/scripts/utils/server.js b/apps/stress-test/scripts/utils/server.js new file mode 100644 index 00000000000000..ac7d42f351f35f --- /dev/null +++ b/apps/stress-test/scripts/utils/server.js @@ -0,0 +1,61 @@ +const express = require('express'); +const { getPackageRoot } = require('./paths'); +const { join } = require('path'); + +/** + * @param {*} arg + */ +const handleGracefulShutdown = arg => { + console.log('Shutting down...'); + if (server && server.close) { + console.log('Closing HTTP server'); + server.close(); + process.off('uncaughtException', handleGracefulShutdown); + } + + if (arg instanceof Error) { + console.error('Error', arg); + process.exit(1); + } +}; + +let server; +/** + * + * @param {CLIServerOptions} options + * @returns + */ +const startServer = (options = {}) => { + return new Promise((resolve, reject) => { + if (server) { + return reject(new Error('Server already running.')); + } + + const { root = 'dist', port = 8080 } = options; + const rootPath = join(getPackageRoot(), root); + + const app = express(); + app.use(express.static(rootPath)); + + process.on('uncaughtException', handleGracefulShutdown); + + server = app.listen(port, () => { + console.log(`Server listening at http://localhost:${port}`); + console.log(`Serving directory: ${rootPath}`); + resolve(); + }); + }); +}; + +const stopServer = () => { + if (!server) { + throw new Error('No server running.'); + } + + server.close(); +}; + +module.exports = { + startServer, + stopServer, +}; diff --git a/apps/stress-test/scripts/utils/tachometer.js b/apps/stress-test/scripts/utils/tachometer.js new file mode 100644 index 00000000000000..13903ca32fb7e1 --- /dev/null +++ b/apps/stress-test/scripts/utils/tachometer.js @@ -0,0 +1,31 @@ +const { exec } = require('child_process'); +const { promisify } = require('util'); +const { ensureClean } = require('./paths'); + +const execAsync = promisify(exec); + +/** + * @function runTachometer + * @param {ConfigResult[]} options + */ +const runTachometer = async testConfigs => { + // Everything will be going to the same results folder for a test run + const { resultsDir } = testConfigs[0]; + ensureClean(resultsDir); + + for (const testConfig of testConfigs) { + const { testFile, resultsFile } = testConfig; + console.log('\n--------------------\n'); + console.log(`Running test case "${testFile}...`); + const cmd = `tach --config ${testFile} --json-file ${resultsFile}`; + + // Make sure we run these in sequence. + // We don't want to run several tests at once as they + // may unduly influence each other. + const res = await execAsync(cmd); + console.log(res.stdout); + console.log(`Test complete! Results written to ${resultsFile}.`); + } +}; + +module.exports = runTachometer; From d923b5b2f7be3acb774f9dd5ad80a9b6b0c57a8f Mon Sep 17 00:00:00 2001 From: Sean Monahan Date: Fri, 26 Aug 2022 15:52:58 -0700 Subject: [PATCH 2/5] add missing 'getBrowsers' script. --- apps/stress-test/scripts/utils/configureYargs.js | 2 +- apps/stress-test/scripts/utils/getBrowsers.js | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 apps/stress-test/scripts/utils/getBrowsers.js diff --git a/apps/stress-test/scripts/utils/configureYargs.js b/apps/stress-test/scripts/utils/configureYargs.js index 85f91d1cecfe5e..2a2fd079c09df3 100644 --- a/apps/stress-test/scripts/utils/configureYargs.js +++ b/apps/stress-test/scripts/utils/configureYargs.js @@ -1,4 +1,4 @@ -const { getBrowsers } = require('../../dist/getBrowsers'); +const { getBrowsers } = require('./getBrowsers'); /** * @typedef {Object. { + // https://github.com/Polymer/tachometer#webdriver-plugins + const browsers = ['chrome', 'firefox' /*'edge'*/]; + + if (os.type() === 'Darwin') { + browsers.push('safari'); + } + + return browsers; +}; From 257953f7b47756c5926c420a90cbc59ca99552f3 Mon Sep 17 00:00:00 2001 From: Sean Monahan Date: Wed, 7 Sep 2022 08:43:28 -0700 Subject: [PATCH 3/5] pr feedback --- apps/stress-test/README.md | 16 ++-- apps/stress-test/package.json | 3 +- .../scripts/commands/buildTestConfig.js | 32 ++++---- .../scripts/commands/processResults.js | 21 +++-- apps/stress-test/scripts/commands/run.js | 20 +++-- .../stress-test/scripts/commands/runServer.js | 26 ++++--- .../scripts/commands/runTachometer.js | 29 ++++--- apps/stress-test/scripts/stressTest.js | 11 +-- apps/stress-test/tsconfig.json | 1 + apps/stress-test/tsconfig.type.json | 7 ++ package.json | 1 - yarn.lock | 77 +------------------ 12 files changed, 102 insertions(+), 142 deletions(-) create mode 100644 apps/stress-test/tsconfig.type.json diff --git a/apps/stress-test/README.md b/apps/stress-test/README.md index 6c707cbabc6857..8e70ac2185a451 100644 --- a/apps/stress-test/README.md +++ b/apps/stress-test/README.md @@ -22,7 +22,7 @@ The project is laid out with folders for each supported version of Fluent (`v8`, The `shared` folder is for utilities that are shared across Fluent versions. -The `components` folder is also split by supported Fluent versions and is where compoenents that can be shared across test cases live. +The `components` folder is also split by supported Fluent versions and is where components that can be shared across test cases live. The `benchmarks` folder houses Tachometer configurations and test results; and helper scripts for generating configurations and processing results. @@ -48,7 +48,9 @@ The CLI is used to run tests and process the results of tests. The CLI handles c Invoke the CLI with - $ yarn workspace @fluentui/stress-test stress-test +```shell +$ yarn workspace @fluentui/stress-test stress-test +``` This will show the help documentation which will provide the details for each command. What follows is a high level overview of the CLI's features. @@ -62,8 +64,10 @@ This will show the help documentation which will provide the details for each co ### Examples - # Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against Firefox with small sizes and low sample size - $ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --sample-size 2 --test-cases mount prop-update --browsers firefox --sizes xs s +```shell +# Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against Firefox with small sizes and low sample size +$ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --sample-size 2 --test-cases mount prop-update --browsers firefox --sizes xs s - # Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against the default browsers at the default sizes and sample size - $ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --test-cases mount prop-update +# Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against the default browsers at the default sizes and sample size +$ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --test-cases mount prop-update +``` diff --git a/apps/stress-test/package.json b/apps/stress-test/package.json index 4f294df751cb26..75bc75e16e6f39 100644 --- a/apps/stress-test/package.json +++ b/apps/stress-test/package.json @@ -8,7 +8,8 @@ "stress-test": "node ./scripts/stressTest.js", "start": "webpack-dev-server --mode=development", "build:local": "lage build --to @fluentui/react @fluentui/web-components --verbose && yarn build:app", - "build:app": "webpack --mode=production" + "build:app": "webpack --mode=production", + "type-check": "tsc -b tsconfig.type.json" }, "dependencies": { "@fluentui/react": "^8.91.1", diff --git a/apps/stress-test/scripts/commands/buildTestConfig.js b/apps/stress-test/scripts/commands/buildTestConfig.js index f68da1a8b26d54..5ecd3350448f11 100644 --- a/apps/stress-test/scripts/commands/buildTestConfig.js +++ b/apps/stress-test/scripts/commands/buildTestConfig.js @@ -24,12 +24,6 @@ const querystring = require('querystring'); */ const command = 'build-test-config'; -exports.command = command; -exports.describe = 'Builds test configuration files.'; - -exports.builder = yargs => { - configureYargs(command, yargs); -}; /** * @function buildTestConfig @@ -112,13 +106,23 @@ const makeConfigJson = (scenario, browser, testCase, sampleSize, targets, size, return JSON.stringify(json, null, 4); }; -/** - * - * @param {CLIBuildTestConfigOptions} argv - */ -exports.handler = argv => { - const options = processOptions(argv); - buildTestConfig(options); +/** @type {import('yargs').CommandModule} */ +const api = { + command, + describe: 'Builds test configuration files.', + builder: yargs => { + configureYargs(command, yargs); + }, + /** + * @param {CLIBuildTestConfigOptions} argv + */ + handler: argv => { + const options = processOptions(argv); + buildTestConfig(options); + }, }; -exports.buildTestConfig = buildTestConfig; +module.exports = { + ...api, + buildTestConfig, +}; diff --git a/apps/stress-test/scripts/commands/processResults.js b/apps/stress-test/scripts/commands/processResults.js index 15910a5c7d9587..d391cddc7a740b 100644 --- a/apps/stress-test/scripts/commands/processResults.js +++ b/apps/stress-test/scripts/commands/processResults.js @@ -9,18 +9,11 @@ const { getResultsDir, readDirJson } = require('../utils/paths'); */ const command = 'process-results'; -exports.command = command; -exports.describe = 'Processes test results for display with charts and graphs.'; - -exports.builder = yargs => { - configureYargs(command, yargs); -}; /** - * * @param {CLIProcessResultsOptions} argv */ -exports.handler = argv => { +const handler = argv => { const { scenario } = argv; const resultsDir = getResultsDir(scenario); @@ -54,3 +47,15 @@ exports.handler = argv => { fs.writeFileSync(path.join(resultsDir, 'processed-results.js'), js, { encoding: 'utf8' }); }; + +/** @type {import('yargs').CommandModule} */ +const api = { + command, + describe: 'Processes test results for display with charts and graphs.', + builder: yargs => { + configureYargs(command, yargs); + }, + handler, +}; + +module.exports = api; diff --git a/apps/stress-test/scripts/commands/run.js b/apps/stress-test/scripts/commands/run.js index 6b4321e3e4f885..da73b50cfbced5 100644 --- a/apps/stress-test/scripts/commands/run.js +++ b/apps/stress-test/scripts/commands/run.js @@ -20,12 +20,6 @@ const { buildTestConfig } = require('./buildTestConfig'); */ const command = 'run'; -exports.command = command; -exports.describe = 'Builds configs and runs stress testing.'; - -exports.builder = yargs => { - configureYargs(command, yargs); -}; /** * @@ -42,7 +36,7 @@ const run = async (testConfigs, options) => { * * @param {CLIRunOptions} argv */ -exports.handler = argv => { +const handler = argv => { const options = processOptions(argv); const testConfigs = buildTestConfig(options); @@ -54,3 +48,15 @@ exports.handler = argv => { } }); }; + +/** @type {import('yargs').CommandModule} */ +const api = { + command, + describe: 'Builds configs and runs stress testing.', + builder: yargs => { + configureYargs(command, yargs); + }, + handler, +}; + +module.exports = api; diff --git a/apps/stress-test/scripts/commands/runServer.js b/apps/stress-test/scripts/commands/runServer.js index 9535cb24997fb3..2c21396b6f4e5c 100644 --- a/apps/stress-test/scripts/commands/runServer.js +++ b/apps/stress-test/scripts/commands/runServer.js @@ -8,17 +8,21 @@ const { startServer } = require('../utils/server'); */ const command = 'serve'; -exports.command = command; -exports.describe = 'Runs a test HTTP server.'; -exports.builder = yargs => { - configureYargs(command, yargs); +/** @type {import('yargs').CommandModule} */ +const api = { + command, + describe: 'Runs a test HTTP server.', + builder: yargs => { + configureYargs(command, yargs); + }, + /** + * + * @param {CLIServerOptions} argv + */ + handler: argv => { + startServer(argv); + }, }; -/** - * - * @param {CLIServerOptions} argv - */ -exports.handler = argv => { - startServer(argv); -}; +module.exports = api; diff --git a/apps/stress-test/scripts/commands/runTachometer.js b/apps/stress-test/scripts/commands/runTachometer.js index 5aec7153f8d388..acdb4ee5eb84e9 100644 --- a/apps/stress-test/scripts/commands/runTachometer.js +++ b/apps/stress-test/scripts/commands/runTachometer.js @@ -11,22 +11,16 @@ const runTachometer = require('../utils/tachometer'); */ const command = 'tachometer'; -exports.command = command; -exports.describe = 'Runs Tachometer for a provided scenario.'; -exports.builder = yargs => { - configureYargs(command, yargs); -}; - -const run = async testConfigs => { - await runTachometer(testConfigs); -}; +// const run = async testConfigs => { +// await runTachometer(testConfigs); +// }; /** * * @param {CLITachometerOptions} argv */ -exports.handler = argv => { +const handler = async argv => { const { scenario } = argv; const configDir = getConfigDir(scenario); @@ -50,5 +44,18 @@ exports.handler = argv => { configs.push(configResult); } - run(configs).then(() => console.log('Tachometer run complete!')); + await runTachometer(configs); + console.log('Tachometer run complete!'); }; + +/** @type {import('yargs').CommandModule} */ +const api = { + command, + describe: 'Runs Tachometer for a provided scenario.', + builder: yargs => { + configureYargs(command, yargs); + }, + handler, +}; + +module.exports = api; diff --git a/apps/stress-test/scripts/stressTest.js b/apps/stress-test/scripts/stressTest.js index 2dbdbd4f231c6c..2fdfb1e4385d75 100644 --- a/apps/stress-test/scripts/stressTest.js +++ b/apps/stress-test/scripts/stressTest.js @@ -1,12 +1,3 @@ const yargs = require('yargs'); -yargs - .usage('Usage: $0 [options]') - .command(require('./commands/run.js')) - .command(require('./commands/buildTestConfig')) - .command(require('./commands/processResults.js')) - .command(require('./commands/runServer.js')) - .command(require('./commands/runTachometer.js')) - .demandCommand() - .help() - .parse(); +yargs.usage('Usage: $0 [options]').commandDir('commands').demandCommand().help().parse(); diff --git a/apps/stress-test/tsconfig.json b/apps/stress-test/tsconfig.json index 0831510410cd52..d03eef02f54b59 100644 --- a/apps/stress-test/tsconfig.json +++ b/apps/stress-test/tsconfig.json @@ -3,6 +3,7 @@ "compilerOptions": { "noEmit": false, "allowJs": true, + "checkJs": true, "importHelpers": true, "isolatedModules": true, "jsx": "react", diff --git a/apps/stress-test/tsconfig.type.json b/apps/stress-test/tsconfig.type.json new file mode 100644 index 00000000000000..02b553992117de --- /dev/null +++ b/apps/stress-test/tsconfig.type.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": true + }, + "include": ["src", "scenarios"] +} diff --git a/package.json b/package.json index 99860546313537..bce0bdefff1cfa 100644 --- a/package.json +++ b/package.json @@ -199,7 +199,6 @@ "gulp": "^4.0.2", "gzip-size": "6.0.0", "html-webpack-plugin": "5.1.0", - "http-server": "14.1.1", "ignore-not-found-export-webpack-plugin": "1.0.2", "imports-loader": "1.2.0", "jest": "26.6.3", diff --git a/yarn.lock b/yarn.lock index dbc741f1066e61..cdbccd3702833a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8183,13 +8183,6 @@ base@^0.11.1: mixin-deep "^1.2.0" pascalcase "^0.1.1" -basic-auth@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.1.tgz#b998279bf47ce38344b4f3cf916d4679bbf51e3a" - integrity sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg== - dependencies: - safe-buffer "5.1.2" - batch-processor@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/batch-processor/-/batch-processor-1.0.0.tgz#75c95c32b748e0850d10c2b168f6bdbe9891ace8" @@ -9116,7 +9109,7 @@ chalk@^3.0.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chalk@^4.0.0, chalk@^4.0.2, chalk@^4.1.0, chalk@^4.1.1, chalk@^4.1.2: +chalk@^4.0.0, chalk@^4.0.2, chalk@^4.1.0, chalk@^4.1.1: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -10224,11 +10217,6 @@ cors@~2.8.5: object-assign "^4" vary "^1" -corser@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/corser/-/corser-2.0.1.tgz#8eda252ecaab5840dcd975ceb90d9370c819ff87" - integrity sha512-utCYNzRSQIZNPIcGZdQc92UVJYAhtGAteCFg0yRaFm8f0P+CPtyGyHXJcGXnffjCybUCEx3FQ2G7U3/o9eIkVQ== - cosmiconfig@7.0.0, cosmiconfig@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.0.0.tgz#ef9b44d773959cae63ddecd122de23853b60f8d3" @@ -15232,13 +15220,6 @@ html-encoding-sniffer@^2.0.1: dependencies: whatwg-encoding "^1.0.5" -html-encoding-sniffer@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz#2cb1a8cf0db52414776e5b2a7a04d5dd98158de9" - integrity sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA== - dependencies: - whatwg-encoding "^2.0.0" - html-entities@^2.1.0, html-entities@^2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-2.3.2.tgz#760b404685cb1d794e4f4b744332e3b00dcfe488" @@ -15509,25 +15490,6 @@ http-proxy@^1.18.1, http-proxy@~1.18.1: follow-redirects "^1.0.0" requires-port "^1.0.0" -http-server@14.1.1: - version "14.1.1" - resolved "https://registry.yarnpkg.com/http-server/-/http-server-14.1.1.tgz#d60fbb37d7c2fdff0f0fbff0d0ee6670bd285e2e" - integrity sha512-+cbxadF40UXd9T01zUHgA+rlo2Bg1Srer4+B4NwIHdaGxAGGv59nYRnGGDJ9LBk7alpS0US+J+bLLdQOOkJq4A== - dependencies: - basic-auth "^2.0.1" - chalk "^4.1.2" - corser "^2.0.1" - he "^1.2.0" - html-encoding-sniffer "^3.0.0" - http-proxy "^1.18.1" - mime "^1.6.0" - minimist "^1.2.6" - opener "^1.5.1" - portfinder "^1.0.28" - secure-compare "3.0.1" - union "~0.5.0" - url-join "^4.0.1" - http-signature@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" @@ -15635,13 +15597,6 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4: dependencies: safer-buffer ">= 2.1.2 < 3" -iconv-lite@0.6.3: - version "0.6.3" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" - integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== - dependencies: - safer-buffer ">= 2.1.2 < 3.0.0" - iconv-lite@^0.6.2: version "0.6.2" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.2.tgz#ce13d1875b0c3a674bd6a04b7f76b01b1b6ded01" @@ -19649,7 +19604,7 @@ mime-types@^2.1.18, mime-types@~2.1.34: dependencies: mime-db "1.52.0" -mime@1.6.0, mime@^1.6.0: +mime@1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== @@ -20974,7 +20929,7 @@ open@^8.0.9, open@^8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" -opener@^1.5.1, opener@^1.5.2: +opener@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598" integrity sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A== @@ -22478,7 +22433,7 @@ qs@^6.10.0, qs@^6.8.0: dependencies: side-channel "^1.0.4" -qs@^6.4.0, qs@^6.5.2: +qs@^6.5.2: version "6.11.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== @@ -24339,11 +24294,6 @@ screener-storybook@0.23.0: screener-runner "~0.13.0" semver "~5.6.0" -secure-compare@3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/secure-compare/-/secure-compare-3.0.1.tgz#f1a0329b308b221fae37b9974f3d578d0ca999e3" - integrity sha512-AckIIV90rPDcBcglUwXPF3kg0P0qmPsPXAj6BBEENQE1p5yA1xfmDJzfi1Tappj37Pv2mVbKpL3Z1T+Nn7k1Qw== - seek-bzip@^1.0.5: version "1.0.6" resolved "https://registry.yarnpkg.com/seek-bzip/-/seek-bzip-1.0.6.tgz#35c4171f55a680916b52a07859ecf3b5857f21c4" @@ -26897,13 +26847,6 @@ union-value@^1.0.0: is-extendable "^0.1.1" set-value "^2.0.1" -union@~0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/union/-/union-0.5.0.tgz#b2c11be84f60538537b846edb9ba266ba0090075" - integrity sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA== - dependencies: - qs "^6.4.0" - uniq@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/uniq/-/uniq-1.0.1.tgz#b31c5ae8254844a3a8281541ce2b04b865a734ff" @@ -27174,11 +27117,6 @@ urix@^0.1.0: resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI= -url-join@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.1.tgz#b642e21a2646808ffa178c4c5fda39844e12cde7" - integrity sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA== - url-loader@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-4.1.1.tgz#28505e905cae158cf07c92ca622d7f237e70a4e2" @@ -27927,13 +27865,6 @@ whatwg-encoding@^1.0.5: dependencies: iconv-lite "0.4.24" -whatwg-encoding@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz#e7635f597fd87020858626805a2729fa7698ac53" - integrity sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg== - dependencies: - iconv-lite "0.6.3" - whatwg-fetch@2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-2.0.4.tgz#dde6a5df315f9d39991aa17621853d720b85566f" From 9d1c11e08300846a4e15d3e841bf5593ea3371aa Mon Sep 17 00:00:00 2001 From: Sean Monahan Date: Wed, 7 Sep 2022 09:05:45 -0700 Subject: [PATCH 4/5] readme updates --- apps/stress-test/README.md | 78 ++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/apps/stress-test/README.md b/apps/stress-test/README.md index 8e70ac2185a451..74fda3509ad787 100644 --- a/apps/stress-test/README.md +++ b/apps/stress-test/README.md @@ -4,7 +4,41 @@ Stress Test is an application for testing Fluent UI components in "stressful" sc This application is configured to support Fluent UI v8, v9 and Web Components. -Stress Test is a simple Webpack application as Webpack allows us to support Fluent v8, v9 and Web Components in a single app while using each version of Fluent natively (that is, the React versions of Fluent are typical React apps, the Web Component version is a typical web component app). Additionally, Griffel, a dependency of Fluent v9 provides some Webpack loaders so Stress Test needs to be able to use those. +## Usage + +Run tests by invoking the CLI: + +```shell +$ yarn workspace @fluentui/stress-test stress-test +``` + +This will show the help documentation that provides details for each command. + +### Building + +Building produces a static application that can be served with a simple HTTP server. + +```shell +# Build all the dependencies for this application (e.g., `@fluentui/react`, `@fluentui/web-components`) along with the application. +$ yarn workspace @fluentui/stress-test build:local + +# Build only this application. +$ yarn workspace @fluentui/stress-test build:app +``` + +> NOTE: `build:local` is much slower, but is required the first time you're building the application or if you've pulled in lots of changes. Use `build:app` if you don't need to build dependencies like `@fluentui/react` as it's much faster. + +### Examples + +```shell +# Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against Firefox with small sizes and low sample size +$ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --sample-size 2 --test-cases mount prop-update --browsers firefox --sizes xs s + +# Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against the default browsers at the default sizes and sample size +$ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --test-cases mount prop-update +``` + +> NOTE: Tests should be run against production builds. While tests can be run against development builds, and this is useful for gathering quick results and debugging, the performance characteristics of development and production builds can differ quite a bit. ## Glossary @@ -30,44 +64,4 @@ The `scripts` folder house the Stress Test CLI app that is used to run tests. ### Adding test cases -Currently, new test cases must be manually added. Add tests cases to the `pages` array in `webpack.config.js` and they will be generated by Webpack. This configuration keeps the different Fluent versions in separate Webpack chunks. The dev server needs to be restarted after changes to the config file. - -## Production - -**NOTE**: Tests should be run against production builds. While tests can be run against development builds, and this is useful for gathering quick results, the performance characteristics of development and production builds can differ quite a bit. - -Building for production creates a static production application that can be served with a simple HTTP server. - -Run `yarn workspace @fluentui/stress-test build:app` to build only this application. If you do not need to build dependencies like `@fluentui/react` this is much faster. - -Run `yarn workspace @fluentui/stress-test build:local` to build all the dependencies for this application (e.g., `@fluentui/react`, `@fluentui/web-components`) along with the application. This is much slower but useful if this it the first time you're building the app or you've pulled in lots of changes. - -## Using the CLI - -The CLI is used to run tests and process the results of tests. The CLI handles creating configuration files to run tests, spins up an HTTP server and runs the actual tests. Each step can be handled individiually or you can use a convenience command to run everything for a test case. - -Invoke the CLI with - -```shell -$ yarn workspace @fluentui/stress-test stress-test -``` - -This will show the help documentation which will provide the details for each command. What follows is a high level overview of the CLI's features. - -### Commands - -- `run`: Runs a test end-to-end. Creates configuration files, starts up an HTTP server, runs the tests, and processes the results. -- `build-test-config`: Generates test configuration files only. -- `process-results`: Processes the results of a test run. -- `serve`: Starts the HTTP server. -- `tachometer`: Runs Tachometers against an HTTP server based. - -### Examples - -```shell -# Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against Firefox with small sizes and low sample size -$ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --sample-size 2 --test-cases mount prop-update --browsers firefox --sizes xs s - -# Run the "simple-stress" scenario with the "mount" and "prop-update" test cases against the default browsers at the default sizes and sample size -$ yarn workspace @fluentui/stress-test stress-test run --scenario simple-stress --test-cases mount prop-update -``` +Add tests cases to the appropriate `src/pages` sub-folder. For example to create a new test, "my test" for Fluent v9 add it to `src/pages/v9/my-test`. Use an existing page as a guide for the files you need to add. Pages are automatically picked up by Webpack when the dev server is started. From c2d3b209932c320d9cb9eb72a826c19b460272d4 Mon Sep 17 00:00:00 2001 From: Sean Monahan Date: Wed, 7 Sep 2022 10:41:23 -0700 Subject: [PATCH 5/5] clean up comments --- apps/stress-test/scripts/commands/run.js | 2 -- apps/stress-test/scripts/commands/runServer.js | 1 - apps/stress-test/scripts/commands/runTachometer.js | 5 ----- 3 files changed, 8 deletions(-) diff --git a/apps/stress-test/scripts/commands/run.js b/apps/stress-test/scripts/commands/run.js index da73b50cfbced5..42722cea7a4749 100644 --- a/apps/stress-test/scripts/commands/run.js +++ b/apps/stress-test/scripts/commands/run.js @@ -22,7 +22,6 @@ const { buildTestConfig } = require('./buildTestConfig'); const command = 'run'; /** - * * @param {ConfigResult[]} testConfigs * @param {CLIServerOptions} options */ @@ -33,7 +32,6 @@ const run = async (testConfigs, options) => { }; /** - * * @param {CLIRunOptions} argv */ const handler = argv => { diff --git a/apps/stress-test/scripts/commands/runServer.js b/apps/stress-test/scripts/commands/runServer.js index 2c21396b6f4e5c..ea9c88e3a9655e 100644 --- a/apps/stress-test/scripts/commands/runServer.js +++ b/apps/stress-test/scripts/commands/runServer.js @@ -17,7 +17,6 @@ const api = { configureYargs(command, yargs); }, /** - * * @param {CLIServerOptions} argv */ handler: argv => { diff --git a/apps/stress-test/scripts/commands/runTachometer.js b/apps/stress-test/scripts/commands/runTachometer.js index acdb4ee5eb84e9..31dc7f2167b413 100644 --- a/apps/stress-test/scripts/commands/runTachometer.js +++ b/apps/stress-test/scripts/commands/runTachometer.js @@ -12,12 +12,7 @@ const runTachometer = require('../utils/tachometer'); const command = 'tachometer'; -// const run = async testConfigs => { -// await runTachometer(testConfigs); -// }; - /** - * * @param {CLITachometerOptions} argv */ const handler = async argv => {