From b50a71cba757307b0a96b29a58b16c69fd9c1590 Mon Sep 17 00:00:00 2001 From: Dmitry Lavrenov Date: Thu, 9 May 2024 15:25:47 +0300 Subject: [PATCH 01/11] Add keys related fixtures and modify help check script --- .../bash/fixtures/help.build-spec.stdout.txt | 102 ++++++++++++++++++ .../help.key.generate-node-key.stdout.txt | 20 ++++ .../fixtures/help.key.generate.stdout.txt | 50 +++++++++ .../bash/fixtures/help.key.insert.stdout.txt | 85 +++++++++++++++ .../help.key.inspect-node-key.stdout.txt | 23 ++++ .../bash/fixtures/help.key.inspect.stdout.txt | 67 ++++++++++++ .../bash/fixtures/help.key.stdout.txt | 15 +++ utils/e2e-tests/bash/tests/base/help.sh | 52 ++++++--- 8 files changed, 398 insertions(+), 16 deletions(-) create mode 100644 utils/e2e-tests/bash/fixtures/help.build-spec.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.key.generate-node-key.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.key.generate.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.key.insert.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.key.inspect-node-key.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.key.inspect.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.key.stdout.txt diff --git a/utils/e2e-tests/bash/fixtures/help.build-spec.stdout.txt b/utils/e2e-tests/bash/fixtures/help.build-spec.stdout.txt new file mode 100644 index 000000000..db9a55017 --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.build-spec.stdout.txt @@ -0,0 +1,102 @@ +Build a chain specification + +Usage: humanode-peer build-spec [OPTIONS] + +Options: + --raw + Force raw genesis storage output + + --disable-default-bootnode + Disable adding the default bootnode to the specification. + + By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the specification when no bootnode exists. + + --chain + Specify the chain specification. + + It can be one of the predefined ones (dev, local, or staging) or it can be a path to a file with the chainspec (such as one exported by the `build-spec` subcommand). + + --dev + Specify the development chain. + + This flag sets `--chain=dev`, `--force-authoring`, `--rpc-cors=all`, `--alice`, and `--tmp` flags, unless explicitly overridden. + + -d, --base-path + Specify custom base path + + -l, --log ... + Sets a custom logging filter. Syntax is `=`, e.g. -lsync=debug. + + Log levels (least to most verbose) are error, warn, info, debug, and trace. By default, all targets log `info`. The global log level can be set with `-l`. + + --detailed-log-output + Enable detailed log output. + + This includes displaying the log target, log level and thread name. + + This is automatically enabled when something is logged with any higher level than `info`. + + --disable-log-color + Disable log color output + + --enable-log-reloading + Enable feature to dynamically update and reload the log filter. + + Be aware that enabling this feature can lead to a performance decrease up to factor six or more. Depending on the global logging level the performance decrease changes. + + The `system_addLogFilter` and `system_resetLogFilter` RPCs will have no effect with this option not being set. + + --tracing-targets + Sets a custom profiling filter. Syntax is the same as for logging: `=` + + --tracing-receiver + Receiver to process tracing messages + + [default: log] + + Possible values: + - log: Output the tracing records using the log + + --node-key + The secret key to use for libp2p networking. + + The value is a string that is parsed according to the choice of `--node-key-type` as follows: + + `ed25519`: The value is parsed as a hex-encoded Ed25519 32 byte secret key, i.e. 64 hex characters. + + The value of this option takes precedence over `--node-key-file`. + + WARNING: Secrets provided as command-line arguments are easily exposed. Use of this option should be limited to development and testing. To use an externally managed secret key, use `--node-key-file` instead. + + --node-key-type + The type of secret key to use for libp2p networking. + + The secret key of the node is obtained as follows: + + * If the `--node-key` option is given, the value is parsed as a secret key according to the type. See the documentation for `--node-key`. + + * If the `--node-key-file` option is given, the secret key is read from the specified file. See the documentation for `--node-key-file`. + + * Otherwise, the secret key is read from a file with a predetermined, type-specific name from the chain-specific network config directory inside the base directory specified by `--base-dir`. If this file does not exist, it is created with a newly generated secret key of the chosen type. + + The node's secret key determines the corresponding public key and hence the node's peer ID in the context of libp2p. + + [default: ed25519] + + Possible values: + - ed25519: Use ed25519 + + --node-key-file + The file from which to read the node's secret key to use for libp2p networking. + + The contents of the file are parsed according to the choice of `--node-key-type` as follows: + + `ed25519`: The file must contain an unencoded 32 byte or hex encoded Ed25519 secret key. + + If the file does not exist, it is created with a newly generated secret key of the chosen type. + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version diff --git a/utils/e2e-tests/bash/fixtures/help.key.generate-node-key.stdout.txt b/utils/e2e-tests/bash/fixtures/help.key.generate-node-key.stdout.txt new file mode 100644 index 000000000..14fe747de --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.key.generate-node-key.stdout.txt @@ -0,0 +1,20 @@ +Generate a random node key, write it to a file or stdout and write the corresponding peer-id to stderr + +Usage: humanode-peer key generate-node-key [OPTIONS] + +Options: + --file + Name of file to save secret key to. + + If not given, the secret key is printed to stdout. + + --bin + The output is in raw binary format. + + If not given, the output is written as an hex encoded string. + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version diff --git a/utils/e2e-tests/bash/fixtures/help.key.generate.stdout.txt b/utils/e2e-tests/bash/fixtures/help.key.generate.stdout.txt new file mode 100644 index 000000000..3d760cda4 --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.key.generate.stdout.txt @@ -0,0 +1,50 @@ +Generate a random account + +Usage: humanode-peer key generate [OPTIONS] + +Options: + -w, --words + The number of words in the phrase to generate. One of 12 (default), 15, 18, 21 and 24 + + --keystore-uri + Specify custom URIs to connect to for keystore-services + + --keystore-path + Specify custom keystore path + + --password-interactive + Use interactive shell for entering the password used by the keystore + + --password + Password used by the keystore. This allows appending an extra user-defined secret to the seed + + --password-filename + File that contains the password used by the keystore + + -n, --network + network address format + + --output-type + output format + + [default: text] + + Possible values: + - json: Output as json + - text: Output as text + + --scheme + cryptography scheme + + [default: sr25519] + + Possible values: + - ed25519: Use ed25519 + - sr25519: Use sr25519 + - ecdsa: Use + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version diff --git a/utils/e2e-tests/bash/fixtures/help.key.insert.stdout.txt b/utils/e2e-tests/bash/fixtures/help.key.insert.stdout.txt new file mode 100644 index 000000000..7518b1c1c --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.key.insert.stdout.txt @@ -0,0 +1,85 @@ +Insert a key to the keystore of a node + +Usage: humanode-peer key insert [OPTIONS] --key-type --scheme + +Options: + --suri + The secret key URI. If the value is a file, the file content is used as URI. If not given, you will be prompted for the URI + + --key-type + Key type, examples: "gran", or "imon" + + --chain + Specify the chain specification. + + It can be one of the predefined ones (dev, local, or staging) or it can be a path to a file with the chainspec (such as one exported by the `build-spec` subcommand). + + --dev + Specify the development chain. + + This flag sets `--chain=dev`, `--force-authoring`, `--rpc-cors=all`, `--alice`, and `--tmp` flags, unless explicitly overridden. + + -d, --base-path + Specify custom base path + + -l, --log ... + Sets a custom logging filter. Syntax is `=`, e.g. -lsync=debug. + + Log levels (least to most verbose) are error, warn, info, debug, and trace. By default, all targets log `info`. The global log level can be set with `-l`. + + --detailed-log-output + Enable detailed log output. + + This includes displaying the log target, log level and thread name. + + This is automatically enabled when something is logged with any higher level than `info`. + + --disable-log-color + Disable log color output + + --enable-log-reloading + Enable feature to dynamically update and reload the log filter. + + Be aware that enabling this feature can lead to a performance decrease up to factor six or more. Depending on the global logging level the performance decrease changes. + + The `system_addLogFilter` and `system_resetLogFilter` RPCs will have no effect with this option not being set. + + --tracing-targets + Sets a custom profiling filter. Syntax is the same as for logging: `=` + + --tracing-receiver + Receiver to process tracing messages + + [default: log] + + Possible values: + - log: Output the tracing records using the log + + --keystore-uri + Specify custom URIs to connect to for keystore-services + + --keystore-path + Specify custom keystore path + + --password-interactive + Use interactive shell for entering the password used by the keystore + + --password + Password used by the keystore. This allows appending an extra user-defined secret to the seed + + --password-filename + File that contains the password used by the keystore + + --scheme + The cryptography scheme that should be used to generate the key out of the given URI + + Possible values: + - ed25519: Use ed25519 + - sr25519: Use sr25519 + - ecdsa: Use + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version diff --git a/utils/e2e-tests/bash/fixtures/help.key.inspect-node-key.stdout.txt b/utils/e2e-tests/bash/fixtures/help.key.inspect-node-key.stdout.txt new file mode 100644 index 000000000..fff223382 --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.key.inspect-node-key.stdout.txt @@ -0,0 +1,23 @@ +Load a node key from a file or stdin and print the corresponding peer-id + +Usage: humanode-peer key inspect-node-key [OPTIONS] + +Options: + --file + Name of file to read the secret key from. + + If not given, the secret key is read from stdin (up to EOF). + + --bin + The input is in raw binary format. + + If not given, the input is read as an hex encoded string. + + -n, --network + This argument is deprecated and has no effect for this command + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version diff --git a/utils/e2e-tests/bash/fixtures/help.key.inspect.stdout.txt b/utils/e2e-tests/bash/fixtures/help.key.inspect.stdout.txt new file mode 100644 index 000000000..3b236f799 --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.key.inspect.stdout.txt @@ -0,0 +1,67 @@ +Gets a public key and a SS58 address from the provided Secret URI + +Usage: humanode-peer key inspect [OPTIONS] [URI] + +Arguments: + [URI] + A Key URI to be inspected. May be a secret seed, secret URI (with derivation paths and password), SS58, public URI or a hex encoded public key. + + If it is a hex encoded public key, `--public` needs to be given as argument. + + If the given value is a file, the file content will be used as URI. + + If omitted, you will be prompted for the URI. + +Options: + --public + Is the given `uri` a hex encoded public key? + + --keystore-uri + Specify custom URIs to connect to for keystore-services + + --keystore-path + Specify custom keystore path + + --password-interactive + Use interactive shell for entering the password used by the keystore + + --password + Password used by the keystore. This allows appending an extra user-defined secret to the seed + + --password-filename + File that contains the password used by the keystore + + -n, --network + network address format + + --output-type + output format + + [default: text] + + Possible values: + - json: Output as json + - text: Output as text + + --scheme + cryptography scheme + + [default: sr25519] + + Possible values: + - ed25519: Use ed25519 + - sr25519: Use sr25519 + - ecdsa: Use + + --expect-public + Expect that `--uri` has the given public key/account-id. + + If `--uri` has any derivations, the public key is checked against the base `uri`, i.e. the `uri` without any derivation applied. However, if `uri` has a password or there is one given by `--password`, it will be used to decrypt `uri` before comparing the public key/account-id. + + If there is no derivation in `--uri`, the public key will be checked against the public key of `--uri` directly. + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version diff --git a/utils/e2e-tests/bash/fixtures/help.key.stdout.txt b/utils/e2e-tests/bash/fixtures/help.key.stdout.txt new file mode 100644 index 000000000..aa001169e --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.key.stdout.txt @@ -0,0 +1,15 @@ +Key management cli utilities + +Usage: humanode-peer key + +Commands: + generate-node-key Generate a random node key, write it to a file or stdout and write the corresponding peer-id to stderr + generate Generate a random account + inspect Gets a public key and a SS58 address from the provided Secret URI + inspect-node-key Load a node key from a file or stdin and print the corresponding peer-id + insert Insert a key to the keystore of a node + help Print this message or the help of the given subcommand(s) + +Options: + -h, --help Print help + -V, --version Print version diff --git a/utils/e2e-tests/bash/tests/base/help.sh b/utils/e2e-tests/bash/tests/base/help.sh index ab4d831a0..85a122efa 100755 --- a/utils/e2e-tests/bash/tests/base/help.sh +++ b/utils/e2e-tests/bash/tests/base/help.sh @@ -3,23 +3,43 @@ set -euo pipefail SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -# This test ensures we have an adequate and working help command. -# It also serves as a snapshot test for the help text, so we can track when -# the help text changes. - -OUTPUT="$("$HUMANODE_PEER_PATH" --help)" -TEMPLATE="$(cat "$SCRIPT_DIR/../../fixtures/help.stdout.txt")" - -DIFF_CMD_ARGS=( - -u - -b - -I '^humanode-peer {{sha}}$' # template - -I '^humanode-peer [0-9a-f]\{40\}$' # expected +# This test ensures we have an adequate and working help command and subcommands. +# It also serves as a snapshot test for the help texts, so we can track when +# the help texts change. + +compare() { + COMMAND="${@:-""}" + FIXTURE="help."${COMMAND// /.}".stdout.txt" + + echo $FIXTURE + + OUTPUT="$("$HUMANODE_PEER_PATH" $COMMAND --help)" + TEMPLATE="$(cat "$SCRIPT_DIR/../../fixtures/$FIXTURE")" + + DIFF_CMD_ARGS=( + -u + -b + -I '^humanode-peer {{sha}}$' # template + -I '^humanode-peer [0-9a-f]\{40\}$' # expected + ) + + if ! DIFF="$(diff "${DIFF_CMD_ARGS[@]}" <(printf '%s' "$TEMPLATE") <(printf '%s' "$OUTPUT"))"; then + printf "Output did not match:\n%s\n" "$DIFF" + exit 1 + fi +} + +COMMANDS=( + "key" + "key generate-node-key" + "key generate" + "key inspect" + "key inspect-node-key" + "key insert" ) -if ! DIFF="$(diff "${DIFF_CMD_ARGS[@]}" <(printf '%s' "$TEMPLATE") <(printf '%s' "$OUTPUT"))"; then - printf "Output did not match:\n%s\n" "$DIFF" - exit 1 -fi +for COMMAND in "${COMMANDS[@]}"; do + compare $COMMAND +done printf "Test succeded\n" >&2 From 42b7fe7e8bc25a1fe3d203bddf8fa91e8f9bab06 Mon Sep 17 00:00:00 2001 From: Dmitry Lavrenov Date: Thu, 9 May 2024 15:41:36 +0300 Subject: [PATCH 02/11] Properly handle base help command --- utils/e2e-tests/bash/tests/base/help.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/utils/e2e-tests/bash/tests/base/help.sh b/utils/e2e-tests/bash/tests/base/help.sh index 85a122efa..e69dadab2 100755 --- a/utils/e2e-tests/bash/tests/base/help.sh +++ b/utils/e2e-tests/bash/tests/base/help.sh @@ -8,13 +8,15 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) # the help texts change. compare() { - COMMAND="${@:-""}" - FIXTURE="help."${COMMAND// /.}".stdout.txt" + SUBCOMMAND="${@:-""}" - echo $FIXTURE + # Replace subcommands spaces by dots to prepare fixture filename. + FIXTURE_FILENAME="help."${SUBCOMMAND// /.}".stdout.txt" + # Avoid having double dots. + FIXTURE_FILENAME=${FIXTURE_FILENAME//../.} - OUTPUT="$("$HUMANODE_PEER_PATH" $COMMAND --help)" - TEMPLATE="$(cat "$SCRIPT_DIR/../../fixtures/$FIXTURE")" + OUTPUT="$("$HUMANODE_PEER_PATH" $SUBCOMMAND --help)" + TEMPLATE="$(cat "$SCRIPT_DIR/../../fixtures/$FIXTURE_FILENAME")" DIFF_CMD_ARGS=( -u @@ -29,7 +31,8 @@ compare() { fi } -COMMANDS=( +SUBCOMMANDS=( + "" "key" "key generate-node-key" "key generate" @@ -38,8 +41,8 @@ COMMANDS=( "key insert" ) -for COMMAND in "${COMMANDS[@]}"; do - compare $COMMAND +for SUBCOMMAND in "${SUBCOMMANDS[@]}"; do + compare $SUBCOMMAND done printf "Test succeded\n" >&2 From 5532af83a7d542ea15b77ba8b99bfddebfc8ada8 Mon Sep 17 00:00:00 2001 From: Dmitry Lavrenov Date: Thu, 9 May 2024 16:05:10 +0300 Subject: [PATCH 03/11] Add rest fixtures --- .../fixtures/help.benchmark.block.stdout.txt | 242 +++++++++++++++ .../help.benchmark.extrinsic.stdout.txt | 245 ++++++++++++++++ .../help.benchmark.machine.stdout.txt | 92 ++++++ .../help.benchmark.overhead.stdout.txt | 276 ++++++++++++++++++ .../fixtures/help.benchmark.pallet.stdout.txt | 214 ++++++++++++++ .../bash/fixtures/help.benchmark.stdout.txt | 16 + .../help.benchmark.storage.stdout.txt | 194 ++++++++++++ .../bash/fixtures/help.check-block.stdout.txt | 219 ++++++++++++++ .../fixtures/help.export-blocks.stdout.txt | 127 ++++++++ .../help.export-embedded-runtime.stdout.txt | 8 + .../fixtures/help.export-state.stdout.txt | 114 ++++++++ .../bash/fixtures/help.frontier-db.stdout.txt | 123 ++++++++ .../fixtures/help.import-blocks.stdout.txt | 222 ++++++++++++++ .../bash/fixtures/help.purge-chain.stdout.txt | 71 +++++ .../bash/fixtures/help.revert.stdout.txt | 104 +++++++ .../bash/fixtures/help.try-runtime.stdout.txt | 7 + utils/e2e-tests/bash/tests/base/help.sh | 17 ++ 17 files changed, 2291 insertions(+) create mode 100644 utils/e2e-tests/bash/fixtures/help.benchmark.block.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.benchmark.extrinsic.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.benchmark.machine.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.benchmark.overhead.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.benchmark.pallet.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.benchmark.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.benchmark.storage.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.check-block.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.export-blocks.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.export-embedded-runtime.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.export-state.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.frontier-db.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.import-blocks.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.purge-chain.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.revert.stdout.txt create mode 100644 utils/e2e-tests/bash/fixtures/help.try-runtime.stdout.txt diff --git a/utils/e2e-tests/bash/fixtures/help.benchmark.block.stdout.txt b/utils/e2e-tests/bash/fixtures/help.benchmark.block.stdout.txt new file mode 100644 index 000000000..163f84ce2 --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.benchmark.block.stdout.txt @@ -0,0 +1,242 @@ +Benchmark the execution time of historic blocks. + +This can be used to verify that blocks do not use more weight than they consumed in their `WeightInfo`. Example: + +Let's say you are on a Substrate chain and want to verify that the first 3 blocks did not use more weight than declared which would otherwise be an issue. To test this with a dev node, first create one with a temp directory: + +$ substrate --dev -d /tmp/my-dev --execution wasm --wasm-execution compiled + +And wait some time to let it produce 3 blocks. Then benchmark them with: + +$ substrate benchmark-block --from 1 --to 3 --dev -d /tmp/my-dev --execution wasm --wasm-execution compiled --pruning archive + +The output will be similar to this: + +Block 1 with 1 tx used 77.34% of its weight ( 5,308,964 of 6,864,645 ns) Block 2 with 1 tx used 77.99% of its weight ( 5,353,992 of 6,864,645 ns) Block 3 with 1 tx used 75.91% of its weight ( 5,305,938 of 6,989,645 ns) + +The percent number is important and indicates how much weight was used as compared to the consumed weight. This number should be below 100% for reference hardware. + +Usage: humanode-peer benchmark block [OPTIONS] --from --to + +Options: + --chain + Specify the chain specification. + + It can be one of the predefined ones (dev, local, or staging) or it can be a path to a file with the chainspec (such as one exported by the `build-spec` subcommand). + + --dev + Specify the development chain. + + This flag sets `--chain=dev`, `--force-authoring`, `--rpc-cors=all`, `--alice`, and `--tmp` flags, unless explicitly overridden. + + -d, --base-path + Specify custom base path + + -l, --log ... + Sets a custom logging filter. Syntax is `=`, e.g. -lsync=debug. + + Log levels (least to most verbose) are error, warn, info, debug, and trace. By default, all targets log `info`. The global log level can be set with `-l`. + + --detailed-log-output + Enable detailed log output. + + This includes displaying the log target, log level and thread name. + + This is automatically enabled when something is logged with any higher level than `info`. + + --disable-log-color + Disable log color output + + --enable-log-reloading + Enable feature to dynamically update and reload the log filter. + + Be aware that enabling this feature can lead to a performance decrease up to factor six or more. Depending on the global logging level the performance decrease changes. + + The `system_addLogFilter` and `system_resetLogFilter` RPCs will have no effect with this option not being set. + + --tracing-targets + Sets a custom profiling filter. Syntax is the same as for logging: `=` + + --tracing-receiver + Receiver to process tracing messages + + [default: log] + + Possible values: + - log: Output the tracing records using the log + + --state-pruning + Specify the state pruning mode. + + This mode specifies when the block's state (ie, storage) should be pruned (ie, removed) from the database. + + This setting can only be set on the first creation of the database. Every subsequent run will load the pruning mode from the database and will error if the stored mode doesn't match this CLI value. It is fine to drop this CLI flag for subsequent runs. + + Possible values: + + - archive: + + Keep the state of all blocks. + + - 'archive-canonical' + + Keep only the state of finalized blocks. + + - number + + Keep the state of the last number of finalized blocks. + + [default: 256] + + --blocks-pruning + Specify the blocks pruning mode. + + This mode specifies when the block's body (including justifications) should be pruned (ie, removed) from the database. + + Possible values: - 'archive' + + Keep all blocks. + + - 'archive-canonical' + + Keep only finalized blocks. + + - number + + Keep the last `number` of finalized blocks. + + [default: archive-canonical] + + --database + Select database backend to use + + Possible values: + - rocksdb: Facebooks RocksDB + - paritydb: ParityDb. + - auto: Detect whether there is an existing database. Use it, if there is, if not, create new instance of ParityDb + - paritydb-experimental: ParityDb. + + --db-cache + Limit the memory the database cache can use + + --wasm-execution + Method for executing Wasm runtime code + + [default: compiled] + + Possible values: + - interpreted-i-know-what-i-do: Uses an interpreter + - compiled: Uses a compiled runtime + + --wasmtime-instantiation-strategy + The WASM instantiation method to use. + + Only has an effect when `wasm-execution` is set to `compiled`. + + The copy-on-write strategies are only supported on Linux. If the copy-on-write variant of a strategy is unsupported the executor will fall back to the non-CoW equivalent. + + The fastest (and the default) strategy available is `pooling-copy-on-write`. + + The `legacy-instance-reuse` strategy is deprecated and will be removed in the future. It should only be used in case of issues with the default instantiation strategy. + + [default: pooling-copy-on-write] + + Possible values: + - pooling-copy-on-write: Pool the instances to avoid initializing everything from scratch on each instantiation. Use copy-on-write memory when possible + - recreate-instance-copy-on-write: Recreate the instance from scratch on every instantiation. Use copy-on-write memory when possible + - pooling: Pool the instances to avoid initializing everything from scratch on each instantiation + - recreate-instance: Recreate the instance from scratch on every instantiation. Very slow + - legacy-instance-reuse: Legacy instance reuse mechanism. DEPRECATED. Will be removed in the future + + --wasm-runtime-overrides + Specify the path where local WASM runtimes are stored. + + These runtimes will override on-chain runtimes when the version matches. + + --execution-syncing + The means of execution used when calling into the runtime for importing blocks as part of an initial sync + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-import-block + The means of execution used when calling into the runtime for general block import (including locally authored blocks) + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-block-construction + The means of execution used when calling into the runtime while constructing blocks + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-offchain-worker + The means of execution used when calling into the runtime while using an off-chain worker + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-other + The means of execution used when calling into the runtime while not syncing, importing or constructing blocks + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution + The execution strategy that should be used by all execution contexts + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --trie-cache-size + Specify the state cache size. + + Providing `0` will disable the cache. + + [default: 67108864] + + --state-cache-size + DEPRECATED + + Switch to `--trie-cache-size`. + + --from + Number of the first block to consider + + --to + Last block number to consider + + --repeat + Number of times that the benchmark should be repeated for each block + + [default: 10] + + --enable-trie-cache + Enable the Trie cache. + + This should only be used for performance analysis and not for final results. + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version diff --git a/utils/e2e-tests/bash/fixtures/help.benchmark.extrinsic.stdout.txt b/utils/e2e-tests/bash/fixtures/help.benchmark.extrinsic.stdout.txt new file mode 100644 index 000000000..252879f30 --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.benchmark.extrinsic.stdout.txt @@ -0,0 +1,245 @@ +Benchmark the execution time of different extrinsics. + +This is calculated by filling a block with a specific extrinsic and executing the block. The result time is then divided by the number of extrinsics in that block. + +NOTE: The BlockExecutionWeight is ignored in this case since it + +Usage: humanode-peer benchmark extrinsic [OPTIONS] + +Options: + --chain + Specify the chain specification. + + It can be one of the predefined ones (dev, local, or staging) or it can be a path to a file with the chainspec (such as one exported by the `build-spec` subcommand). + + --dev + Specify the development chain. + + This flag sets `--chain=dev`, `--force-authoring`, `--rpc-cors=all`, `--alice`, and `--tmp` flags, unless explicitly overridden. + + -d, --base-path + Specify custom base path + + -l, --log ... + Sets a custom logging filter. Syntax is `=`, e.g. -lsync=debug. + + Log levels (least to most verbose) are error, warn, info, debug, and trace. By default, all targets log `info`. The global log level can be set with `-l`. + + --detailed-log-output + Enable detailed log output. + + This includes displaying the log target, log level and thread name. + + This is automatically enabled when something is logged with any higher level than `info`. + + --disable-log-color + Disable log color output + + --enable-log-reloading + Enable feature to dynamically update and reload the log filter. + + Be aware that enabling this feature can lead to a performance decrease up to factor six or more. Depending on the global logging level the performance decrease changes. + + The `system_addLogFilter` and `system_resetLogFilter` RPCs will have no effect with this option not being set. + + --tracing-targets + Sets a custom profiling filter. Syntax is the same as for logging: `=` + + --tracing-receiver + Receiver to process tracing messages + + [default: log] + + Possible values: + - log: Output the tracing records using the log + + --state-pruning + Specify the state pruning mode. + + This mode specifies when the block's state (ie, storage) should be pruned (ie, removed) from the database. + + This setting can only be set on the first creation of the database. Every subsequent run will load the pruning mode from the database and will error if the stored mode doesn't match this CLI value. It is fine to drop this CLI flag for subsequent runs. + + Possible values: + + - archive: + + Keep the state of all blocks. + + - 'archive-canonical' + + Keep only the state of finalized blocks. + + - number + + Keep the state of the last number of finalized blocks. + + [default: 256] + + --blocks-pruning + Specify the blocks pruning mode. + + This mode specifies when the block's body (including justifications) should be pruned (ie, removed) from the database. + + Possible values: - 'archive' + + Keep all blocks. + + - 'archive-canonical' + + Keep only finalized blocks. + + - number + + Keep the last `number` of finalized blocks. + + [default: archive-canonical] + + --database + Select database backend to use + + Possible values: + - rocksdb: Facebooks RocksDB + - paritydb: ParityDb. + - auto: Detect whether there is an existing database. Use it, if there is, if not, create new instance of ParityDb + - paritydb-experimental: ParityDb. + + --db-cache + Limit the memory the database cache can use + + --wasm-execution + Method for executing Wasm runtime code + + [default: compiled] + + Possible values: + - interpreted-i-know-what-i-do: Uses an interpreter + - compiled: Uses a compiled runtime + + --wasmtime-instantiation-strategy + The WASM instantiation method to use. + + Only has an effect when `wasm-execution` is set to `compiled`. + + The copy-on-write strategies are only supported on Linux. If the copy-on-write variant of a strategy is unsupported the executor will fall back to the non-CoW equivalent. + + The fastest (and the default) strategy available is `pooling-copy-on-write`. + + The `legacy-instance-reuse` strategy is deprecated and will be removed in the future. It should only be used in case of issues with the default instantiation strategy. + + [default: pooling-copy-on-write] + + Possible values: + - pooling-copy-on-write: Pool the instances to avoid initializing everything from scratch on each instantiation. Use copy-on-write memory when possible + - recreate-instance-copy-on-write: Recreate the instance from scratch on every instantiation. Use copy-on-write memory when possible + - pooling: Pool the instances to avoid initializing everything from scratch on each instantiation + - recreate-instance: Recreate the instance from scratch on every instantiation. Very slow + - legacy-instance-reuse: Legacy instance reuse mechanism. DEPRECATED. Will be removed in the future + + --wasm-runtime-overrides + Specify the path where local WASM runtimes are stored. + + These runtimes will override on-chain runtimes when the version matches. + + --execution-syncing + The means of execution used when calling into the runtime for importing blocks as part of an initial sync + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-import-block + The means of execution used when calling into the runtime for general block import (including locally authored blocks) + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-block-construction + The means of execution used when calling into the runtime while constructing blocks + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-offchain-worker + The means of execution used when calling into the runtime while using an off-chain worker + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-other + The means of execution used when calling into the runtime while not syncing, importing or constructing blocks + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution + The execution strategy that should be used by all execution contexts + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --trie-cache-size + Specify the state cache size. + + Providing `0` will disable the cache. + + [default: 67108864] + + --state-cache-size + DEPRECATED + + Switch to `--trie-cache-size`. + + --warmup + Rounds of warmups before measuring + + [default: 10] + + --repeat + How many times the benchmark should be repeated + + [default: 100] + + --max-ext-per-block + Maximal number of extrinsics that should be put into a block. + + Only useful for debugging. + + --list + List all available pallets and extrinsics. + + The format is CSV with header `pallet, extrinsic`. + + --pallet + Pallet name of the extrinsic to benchmark + + --extrinsic + Extrinsic to benchmark + + --enable-trie-cache + Enable the Trie cache. + + This should only be used for performance analysis and not for final results. + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version diff --git a/utils/e2e-tests/bash/fixtures/help.benchmark.machine.stdout.txt b/utils/e2e-tests/bash/fixtures/help.benchmark.machine.stdout.txt new file mode 100644 index 000000000..82b07b34d --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.benchmark.machine.stdout.txt @@ -0,0 +1,92 @@ +Command to benchmark the hardware. + +Runs multiple benchmarks and prints their output to console. Can be used to gauge if the hardware is fast enough to keep up with a chain's requirements. This command must be integrated by the client since the client can set compiler flags which influence the results. + +You can use the `--base-path` flag to set a location for the disk benchmarks. + +Usage: humanode-peer benchmark machine [OPTIONS] + +Options: + --chain + Specify the chain specification. + + It can be one of the predefined ones (dev, local, or staging) or it can be a path to a file with the chainspec (such as one exported by the `build-spec` subcommand). + + --dev + Specify the development chain. + + This flag sets `--chain=dev`, `--force-authoring`, `--rpc-cors=all`, `--alice`, and `--tmp` flags, unless explicitly overridden. + + -d, --base-path + Specify custom base path + + -l, --log ... + Sets a custom logging filter. Syntax is `=`, e.g. -lsync=debug. + + Log levels (least to most verbose) are error, warn, info, debug, and trace. By default, all targets log `info`. The global log level can be set with `-l`. + + --detailed-log-output + Enable detailed log output. + + This includes displaying the log target, log level and thread name. + + This is automatically enabled when something is logged with any higher level than `info`. + + --disable-log-color + Disable log color output + + --enable-log-reloading + Enable feature to dynamically update and reload the log filter. + + Be aware that enabling this feature can lead to a performance decrease up to factor six or more. Depending on the global logging level the performance decrease changes. + + The `system_addLogFilter` and `system_resetLogFilter` RPCs will have no effect with this option not being set. + + --tracing-targets + Sets a custom profiling filter. Syntax is the same as for logging: `=` + + --tracing-receiver + Receiver to process tracing messages + + [default: log] + + Possible values: + - log: Output the tracing records using the log + + --allow-fail + Do not return an error if any check fails. + + Should only be used for debugging. + + --tolerance + Set a fault tolerance for passing a requirement. + + 10% means that the test would pass even when only 90% score was archived. Can be used to mitigate outliers of the benchmarks. + + [default: 10] + + --verify-duration + Time limit for the verification benchmark + + [default: 5] + + --hash-duration + Time limit for the hash function benchmark + + [default: 5] + + --memory-duration + Time limit for the memory benchmark + + [default: 5] + + --disk-duration + Time limit for each disk benchmark + + [default: 5] + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version diff --git a/utils/e2e-tests/bash/fixtures/help.benchmark.overhead.stdout.txt b/utils/e2e-tests/bash/fixtures/help.benchmark.overhead.stdout.txt new file mode 100644 index 000000000..464bbe9c3 --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.benchmark.overhead.stdout.txt @@ -0,0 +1,276 @@ +Benchmark the execution overhead per-block and per-extrinsic + +Usage: humanode-peer benchmark overhead [OPTIONS] + +Options: + --chain + Specify the chain specification. + + It can be one of the predefined ones (dev, local, or staging) or it can be a path to a file with the chainspec (such as one exported by the `build-spec` subcommand). + + --dev + Specify the development chain. + + This flag sets `--chain=dev`, `--force-authoring`, `--rpc-cors=all`, `--alice`, and `--tmp` flags, unless explicitly overridden. + + -d, --base-path + Specify custom base path + + -l, --log ... + Sets a custom logging filter. Syntax is `=`, e.g. -lsync=debug. + + Log levels (least to most verbose) are error, warn, info, debug, and trace. By default, all targets log `info`. The global log level can be set with `-l`. + + --detailed-log-output + Enable detailed log output. + + This includes displaying the log target, log level and thread name. + + This is automatically enabled when something is logged with any higher level than `info`. + + --disable-log-color + Disable log color output + + --enable-log-reloading + Enable feature to dynamically update and reload the log filter. + + Be aware that enabling this feature can lead to a performance decrease up to factor six or more. Depending on the global logging level the performance decrease changes. + + The `system_addLogFilter` and `system_resetLogFilter` RPCs will have no effect with this option not being set. + + --tracing-targets + Sets a custom profiling filter. Syntax is the same as for logging: `=` + + --tracing-receiver + Receiver to process tracing messages + + [default: log] + + Possible values: + - log: Output the tracing records using the log + + --state-pruning + Specify the state pruning mode. + + This mode specifies when the block's state (ie, storage) should be pruned (ie, removed) from the database. + + This setting can only be set on the first creation of the database. Every subsequent run will load the pruning mode from the database and will error if the stored mode doesn't match this CLI value. It is fine to drop this CLI flag for subsequent runs. + + Possible values: + + - archive: + + Keep the state of all blocks. + + - 'archive-canonical' + + Keep only the state of finalized blocks. + + - number + + Keep the state of the last number of finalized blocks. + + [default: 256] + + --blocks-pruning + Specify the blocks pruning mode. + + This mode specifies when the block's body (including justifications) should be pruned (ie, removed) from the database. + + Possible values: - 'archive' + + Keep all blocks. + + - 'archive-canonical' + + Keep only finalized blocks. + + - number + + Keep the last `number` of finalized blocks. + + [default: archive-canonical] + + --database + Select database backend to use + + Possible values: + - rocksdb: Facebooks RocksDB + - paritydb: ParityDb. + - auto: Detect whether there is an existing database. Use it, if there is, if not, create new instance of ParityDb + - paritydb-experimental: ParityDb. + + --db-cache + Limit the memory the database cache can use + + --wasm-execution + Method for executing Wasm runtime code + + [default: compiled] + + Possible values: + - interpreted-i-know-what-i-do: Uses an interpreter + - compiled: Uses a compiled runtime + + --wasmtime-instantiation-strategy + The WASM instantiation method to use. + + Only has an effect when `wasm-execution` is set to `compiled`. + + The copy-on-write strategies are only supported on Linux. If the copy-on-write variant of a strategy is unsupported the executor will fall back to the non-CoW equivalent. + + The fastest (and the default) strategy available is `pooling-copy-on-write`. + + The `legacy-instance-reuse` strategy is deprecated and will be removed in the future. It should only be used in case of issues with the default instantiation strategy. + + [default: pooling-copy-on-write] + + Possible values: + - pooling-copy-on-write: Pool the instances to avoid initializing everything from scratch on each instantiation. Use copy-on-write memory when possible + - recreate-instance-copy-on-write: Recreate the instance from scratch on every instantiation. Use copy-on-write memory when possible + - pooling: Pool the instances to avoid initializing everything from scratch on each instantiation + - recreate-instance: Recreate the instance from scratch on every instantiation. Very slow + - legacy-instance-reuse: Legacy instance reuse mechanism. DEPRECATED. Will be removed in the future + + --wasm-runtime-overrides + Specify the path where local WASM runtimes are stored. + + These runtimes will override on-chain runtimes when the version matches. + + --execution-syncing + The means of execution used when calling into the runtime for importing blocks as part of an initial sync + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-import-block + The means of execution used when calling into the runtime for general block import (including locally authored blocks) + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-block-construction + The means of execution used when calling into the runtime while constructing blocks + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-offchain-worker + The means of execution used when calling into the runtime while using an off-chain worker + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution-other + The means of execution used when calling into the runtime while not syncing, importing or constructing blocks + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --execution + The execution strategy that should be used by all execution contexts + + Possible values: + - native: Execute with native build (if available, WebAssembly otherwise) + - wasm: Only execute with the WebAssembly build + - both: Execute with both native (where available) and WebAssembly builds + - native-else-wasm: Execute with the native build if possible; if it fails, then execute with WebAssembly + + --trie-cache-size + Specify the state cache size. + + Providing `0` will disable the cache. + + [default: 67108864] + + --state-cache-size + DEPRECATED + + Switch to `--trie-cache-size`. + + --weight-path + File or directory to write the *weight* files to. + + For Substrate this should be `frame/support/src/weights`. + + --metric + Select a specific metric to calculate the final weight output + + [default: average] + + --mul + Multiply the resulting weight with the given factor. Must be positive. + + Is applied before `weight_add`. + + [default: 1] + + --add + Add the given offset to the resulting weight. + + Is applied after `weight_mul`. + + [default: 0] + + --warmup + Rounds of warmups before measuring + + [default: 10] + + --repeat + How many times the benchmark should be repeated + + [default: 100] + + --max-ext-per-block + Maximal number of extrinsics that should be put into a block. + + Only useful for debugging. + + --hostname-override + Manually override the hostname to use + + --hostname-fallback + Specify a fallback hostname if no-one could be detected automatically. + + Note: This only exists to make the `hostname` function infallible. + + [default: ] + + --cpuname-fallback + Specify a fallback CPU name if no-one could be detected automatically. + + Note: This only exists to make the `cpuname` function infallible. + + [default: ] + + --header + Add a header to the generated weight output file. + + Good for adding LICENSE headers. + + --enable-trie-cache + Enable the Trie cache. + + This should only be used for performance analysis and not for final results. + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version diff --git a/utils/e2e-tests/bash/fixtures/help.benchmark.pallet.stdout.txt b/utils/e2e-tests/bash/fixtures/help.benchmark.pallet.stdout.txt new file mode 100644 index 000000000..1ce080489 --- /dev/null +++ b/utils/e2e-tests/bash/fixtures/help.benchmark.pallet.stdout.txt @@ -0,0 +1,214 @@ +Benchmark the extrinsic weight of FRAME Pallets + +Usage: humanode-peer benchmark pallet [OPTIONS] + +Options: + -p, --pallet + Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`) + + -e, --extrinsic + Select an extrinsic inside the pallet to benchmark, or `*` for all + + -s, --steps + Select how many samples we should take across the variable components + + [default: 2] + + --low + Indicates lowest values for each of the component ranges + + --high + Indicates highest values for each of the component ranges + + -r, --repeat + Select how many repetitions of this benchmark should run from within the wasm + + [default: 1] + + --external-repeat + Select how many repetitions of this benchmark should run from the client. + + NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory. + + [default: 1] + + --json + Print the raw results in JSON format + + --json-file + Write the raw results in JSON format into the given file + + --no-median-slopes + Don't print the median-slopes linear regression analysis + + --no-min-squares + Don't print the min-squares linear regression analysis + + --output + Output the benchmarks to a Rust file at the given path + + --header
+ Add a header file to your outputted benchmarks + + --template