diff --git a/.gitignore b/.gitignore index 5c8059f..7ad1f8d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,10 +17,14 @@ hash-sig-keys/ # Auto-generated Ansible inventory (generated from validator-config.yaml) hosts.yml +hosts-prepare.yml # Temporary cache files tmp/ +logs/ +monitoring/ + ansible-deployment*.log # Generated Prometheus config (created by generate-prometheus-config.sh) diff --git a/README.md b/README.md index 53aa989..ce01fd2 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ Every Ansible deployment automatically deploys an observability stack alongside - If specified, uses `root` user for SSH connections - Example: `--useRoot` to connect as root user 10. `--tag` specifies the Docker image tag to use for zeam, ream, qlean, lantern, lighthouse, grandine and ethlambda containers. - - If provided, all clients will use this tag (e.g., `blockblaz/zeam:${tag}`, `ghcr.io/reamlabs/ream:${tag}`, `qdrvm/qlean-mini:${tag}`, `piertwo/lantern:${tag}`, `hopinheimer/lighthouse:${tag}`, `sifrai/grandine:${tag}`, `ghcr.io/lambdaclass/ethlambda:${tag}`) + - If provided, all clients will use this tag (e.g., `blockblaz/zeam:${tag}`, `ghcr.io/reamlabs/ream:${tag}`, `qdrvm/qlean-mini:${tag}`, `piertwo/lantern:${tag}`, `hopinheimer/lighthouse:${tag}`, `sifrai/lean:${tag}`, `ghcr.io/lambdaclass/ethlambda:${tag}`) - If not provided, defaults to `latest` for zeam, ream, and lantern, and `dd67521` for qlean - The script will automatically pull the specified Docker images before running containers - Example: `--tag devnet0` or `--tag devnet1` @@ -230,12 +230,14 @@ Every Ansible deployment automatically deploys an observability stack alongside - Installs: `python3` (Ansible requirement), Docker CE + Compose plugin (all clients run as containers), `yq` (required by the `common` role at every deploy) - Opens per-node ports (`quicPort`/UDP, `metricsPort`/TCP, `apiPort`/TCP) read from the active validator config, plus fixed observability ports (9090, 9080, 9098, 9100). With `--subnets N`, all N nodes' port ranges are opened per host. Enables `ufw` with default deny incoming (persisted across reboots). - Prints a per-tool, per-host status summary (`✅ ok` / `❌ missing`) and `ufw status verbose` - - `--node` is not required; passing unsupported flags alongside `--prepare` produces a prominent error — only `--sshKey` and `--useRoot` are accepted + - **Allowed with `--prepare`:** `--validatorConfig` (path to the template you deploy from), `--subnets N` (so the expanded config and firewall match deploy), `--sshKey` / `--private-key`, `--useRoot`, `--deploymentMode ansible`, `--network`, `--dry-run`, `--logs`, and `NETWORK_DIR`. Other deploy-only flags (`--node`, `--generateGenesis`, `--stop`, …) are rejected. - Example: `NETWORK_DIR=ansible-devnet ./spin-node.sh --prepare --sshKey ~/.ssh/id_ed25519 --useRoot` + - Example with a custom template and subnets: `NETWORK_DIR=ansible-devnet ./spin-node.sh --prepare --subnets 3 --validatorConfig ansible-devnet/genesis/test-validator-config.yaml --sshKey ~/.ssh/id_ed25519 --useRoot` 16. `--subnets N` expand the validator config to deploy N nodes of each client on the same server, where N is 1–5. - - Generates `validator-config-subnets-N.yaml` from the template (without modifying the original) - - Each subnet node gets a unique name (`{client}_0`, `{client}_1`, …), ports incremented by the subnet index, and a fresh P2P identity key for subnets > 0 - - Subnet assignment rule: each server contributes **exactly one node per subnet** — nodes on the same server are never in the same subnet + - **Skipped automatically** when the file's `config.attestation_committee_count` already >= N (the config is already set up for that many subnets). Only needed for template configs where `attestation_committee_count` is 1 or omitted. + - Writes `validator-config-expanded.yaml` under the network genesis dir (without modifying the original template; the file is overwritten on each run with the chosen N) + - Each subnet node gets a unique name (`{client}_0`, `{client}_1`, …), ports offset by `i × number_of_template_rows` (collision-free even on localhost), and a fresh P2P identity key for subnets > 0 + - Subnet assignment rule: each client type appears exactly once per subnet - Every subnet contains the same set of client types - `N=1` renames nodes to `{client}_0` with no port changes (useful for canonical naming) - Example: `NETWORK_DIR=ansible-devnet ./spin-node.sh --node all --subnets 3 --sshKey ~/.ssh/id_ed25519 --useRoot` @@ -249,11 +251,12 @@ Every Ansible deployment automatically deploys an observability stack alongside - `tmp/local-run-DD-MM-YYYY-HH-MM.log` for local deployments - `tmp/ansible-run-DD-MM-YYYY-HH-MM.log` for Ansible deployments - Example: `NETWORK_DIR=local-devnet ./spin-node.sh --node all --logs` -19. `--network` sets the network name label attached to every metric and log stream scraped by the observability stack (Ansible mode only). - - Default: `devnet-3`, set in `parse-env.sh` after argument parsing - - Propagated to Ansible as the `network_name` variable, which is used in `prometheus.yml.j2` and `promtail.yml.j2` templates - - Appears as the `network` label on all Prometheus scrape targets (app, node_exporter, cadvisor) and all Promtail log streams, so you can filter by network in Grafana across multiple environments - - Example: `--network devnet-x` +19. `--network` sets the network name label attached to every metric and log stream scraped by the observability stack. + - **Required for Ansible deployments** — the script exits with an error if omitted when `deployment_mode: ansible` + - For local deployments, falls back to the default network name if not specified + - Propagated to Ansible as the `network_name` variable, used in `prometheus.yml.j2` and `promtail.yml.j2` templates + - Appears as the `network` label on all Prometheus scrape targets and Promtail log streams, so you can filter by network in Grafana + - Example: `--network ` ### Preparing remote servers @@ -275,8 +278,8 @@ NETWORK_DIR=ansible-devnet ./spin-node.sh --prepare --sshKey ~/.ssh/id_ed25519 - **Constraints:** - Only works in ansible mode (`deployment_mode: ansible` in your config, or `--deploymentMode ansible`) -- Passing unsupported flags (e.g. `--node`, `--generateGenesis`) alongside `--prepare` produces a prominent error — only `--sshKey` and `--useRoot` are accepted -- `--node` is not required; the playbook runs on all remote hosts in the inventory +- Passing deploy-only flags (e.g. `--node`, `--generateGenesis`, `--stop`, `--metrics`) alongside `--prepare` produces a prominent error. Use `--validatorConfig`, `--subnets N`, `--sshKey`, `--useRoot`, `--deploymentMode ansible`, `--network`, `--dry-run`, or `--logs` when needed so inventory and firewall match your deploy. +- `--node` is not required; the prepare playbook runs on **one play per unique IP** (deduplicated inventory) so parallel prepares do not fight over the same host Once preparation succeeds, proceed with the normal deploy command: @@ -286,30 +289,68 @@ NETWORK_DIR=ansible-devnet ./spin-node.sh --node all --generateGenesis --sshKey ### Deploying multiple subnets -Use `--subnets N` to run N independent copies of each client on the same server. This is useful for testing multi-subnet P2P scenarios without provisioning additional machines. +There are two ways to run a multi-subnet devnet: **template expansion** (let the script generate nodes) or **hand-maintained config** (you list every node yourself). + +#### Template config vs. expanded/hand-maintained config + +A **template** is a compact `validator-config.yaml` with one row per client (or per host), and `attestation_committee_count` set to **1** (or omitted). You pass `--subnets N` and the script generates `validator-config-expanded.yaml` with N nodes per client, incrementing ports and generating fresh P2P keys. The original file is never modified. See `ansible-devnet/genesis/test-validator-config.yaml` for an example template. + +An **expanded** or **hand-maintained** config lists every node explicitly — one row per running process — with `attestation_committee_count` already set to the target value (e.g. 4). These files need no expansion; do **not** pass `--subnets` (or if you do, it is automatically skipped). See `ansible-devnet/genesis/validator-config.yaml` for an example. + +**How the script decides whether to expand:** + +| `attestation_committee_count` in file | `--subnets N` | Behavior | +|---|---|---| +| Missing or **1** | `--subnets N` (N > 1) | Expansion runs, generates `validator-config-expanded.yaml` | +| **K** (K >= N) | `--subnets N` | Expansion **skipped** — config already covers N subnets | +| **K** (K >= 1) | *(not passed)* | File used as-is, no expansion | + +In short: `--subnets` takes precedence only when it **exceeds** the file's `attestation_committee_count`. + +#### Deploying with a hand-maintained config (no `--subnets`) + +If you maintain your own config with all nodes listed and `attestation_committee_count` already set, deploy directly without `--subnets`: + +```sh +# Hand-maintained config with attestation_committee_count: 4 and 16 nodes +NETWORK_DIR=ansible-devnet ./spin-node.sh --node all --generateGenesis \ + --validatorConfig ~/my-devnet4-config.yaml \ + --sshKey ~/.ssh/id_ed25519 --useRoot --network devnet-4 +``` + +#### Expansion modes + +When expansion does run, `generate-subnet-config.py` picks one of two layouts based on whether each client type appears exactly once in the template. The repository includes an example expanded file at `ansible-devnet/genesis/validator-config-expanded.yaml` (regenerate locally when you change the template or `N`). + +**A — Replicate mode (each client type appears exactly once)** +Use this when each client type has one template row. Every row is cloned **N** times (names `client_0` … `client_{N-1}`), ports are offset by `i × number_of_template_rows`, and subnet 1+ get new P2P keys. Works for both Ansible deployments (unique IPs) and local devnets (all clients on `127.0.0.1`); the stride-based offset prevents port collisions in either case. ```sh # Deploy 3 subnets of every client (ansible) NETWORK_DIR=ansible-devnet ./spin-node.sh --node all --subnets 3 \ --generateGenesis --sshKey ~/.ssh/id_ed25519 --useRoot -``` - -**How it works:** -`--subnets N` generates `validator-config-subnets-N.yaml` from the template (the original file is never modified). For each client in the template it creates N entries: +# Deploy 3 subnets of every client (local devnet) +NETWORK_DIR=local-devnet ./spin-node.sh --node all --subnets 3 --generateGenesis +``` | Subnet index | Name | quicPort | metricsPort | apiPort | |---|---|---|---|---| | 0 | `zeam_0` | base | base | base | -| 1 | `zeam_1` | base+1 | base+1 | base+1 | +| 1 | `zeam_1` | base + stride | base + stride | base + stride | | … | … | … | … | … | -| N-1 | `zeam_N-1` | base+N-1 | base+N-1 | base+N-1 | +| N-1 | `zeam_{N-1}` | base + (N-1)×stride | base + (N-1)×stride | base + (N-1)×stride | -**Rules enforced:** -- `N` must be between 1 and 5 -- Each server contributes exactly one node per subnet (nodes on the same server are never in the same subnet) -- Every subnet contains the same set of client types -- Each node beyond subnet 0 gets a fresh P2P identity key +Where `stride = number of rows in the template`. With 9 clients on localhost and N=3, subnet 1 starts at base+9 and subnet 2 at base+18, so no two processes share a port. + +**B — Shared-host mode (duplicate client types in the template)** +Use this when several validator rows share the same client type (e.g. `zeam_0..zeam_4` already listed). The template is **not** cloned: **one output row per input row**. Subnet index comes from each row’s **`subnet`** field, or—if only **one** client type uses that IP—from the numeric suffix in **`name`** (`zeam_0` → subnet 0, `zeam_4` → subnet 4). If **more than one client type** shares an IP (e.g. zeam and ream on the same host), every row for that IP **must** set an explicit integer **`subnet`** (0 … N-1). Ports and keys stay as you defined them; you must avoid collisions. + +**Rules enforced (both modes):** +- `N` must be between 1 and 5; every subnet index used must satisfy `0 <= subnet < N` +- Replicate mode: each client type must appear at most once in the template +- Shared-host mode: for a given IP, each subnet index appears at most once; the same client cannot appear twice in the same subnet on the same IP +- Replicate mode only: each node beyond subnet 0 gets a fresh P2P identity key **Running `--prepare` with subnets:** @@ -427,6 +468,7 @@ shuffle: roundrobin config: activeEpoch: 18 # Required: Exponent for active epochs (2^18 = 262,144 signatures) keyType: "hash-sig" # Required: Network-wide signature scheme (hash-sig for post-quantum security) + attestation_committee_count: 1 # Optional; defaults to 1 (leanSpec chain config) validators: # validator nodes specification - name: "zeam_0" # a 0rth zeam node privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" @@ -441,13 +483,14 @@ validators: # validator nodes specification - `shuffle`: Validator assignment (to nodes) shuffle algorithm (e.g., `roundrobin`) - `config.activeEpoch`: Exponent for active epochs used in hash-sig key generation (2^activeEpoch signatures per active period) - `config.keyType`: Network-wide signature scheme - must be `"hash-sig"` for post-quantum security +- `config.attestation_committee_count` (optional): Written to `config.yaml` as `ATTESTATION_COMMITTEE_COUNT` (default **1** if omitted, matching leanSpec `ATTESTATION_COMMITTEE_COUNT`) ### Step 1 - Genesis Generation The `spin-node.sh` triggers genesis generator (`generate-genesis.sh`) which generates the following files based on `validator-config.yaml`: 1. **post-quantum secure validator keypairs** in `genesis/hash-sig-keys` unless already generated or forced with `--forceKeyGen` -2. **config.yaml** - With the updated genesis time in short future and pubkeys of the generated keypairs +2. **config.yaml** - Updated genesis time, `ATTESTATION_COMMITTEE_COUNT`, and `GENESIS_VALIDATORS` with **attestation** and **proposal** public keys per validator (dual-key layout / `hash-sig-cli:devnet4`) 3. **validators.yaml** - Validator index assignments using round-robin distribution 4. **nodes.yaml** - ENR (Ethereum Node Records) for peer discovery 5. **genesis.json** - Genesis state in JSON format @@ -469,21 +512,26 @@ You can also run the generator standalone: #### Hash-Based Signature (Post-Quantum) Scheme Validator Keys -**Tool's Docker Image**: `HASH_SIG_CLI_IMAGE="blockblaz/hash-sig-cli:latest"` +**Tool's Docker Image**: `HASH_SIG_CLI_IMAGE="blockblaz/hash-sig-cli:devnet4"` **Source**: https://github.com/blockblaz/hash-sig-cli Using the above docker tool the following files are generated (unless already generated or forced via `--forceKeyGen` flag): -**Generated files:** +**Generated files (dual-key manifest — two roles per validator index):** ``` local-devnet/genesis/hash-sig-keys/ -├── validator-keys-manifest.yaml # Metadata for all keys -├── validator_0_pk.json # Public key for validator 0 -├── validator_0_sk.json # Secret key for validator 0 -├── validator_1_pk.json # Public key for validator 1 -├── validator_1_sk.json # Secret key for validator 1 -└── ... # Keys for additional validators +├── validator-keys-manifest.yaml # Metadata (attester + proposer pubkeys per index) +├── validator_0_proposer_key_pk.json # Proposer public key (validator 0) +├── validator_0_proposer_key_sk.json # Proposer secret key +├── validator_0_attester_key_pk.json # Attester (attestation) public key +├── validator_0_attester_key_sk.json # Attester secret key +├── validator_1_proposer_key_pk.json +├── validator_1_proposer_key_sk.json +├── validator_1_attester_key_pk.json +├── validator_1_attester_key_sk.json +└── ... # Same pattern for additional validators ``` +Older **single-key** layouts (`validator_N_pk.json` / `validator_N_sk.json` only) are still recognized when regenerating from an existing manifest. **Signature Scheme:** The system uses the **SIGTopLevelTargetSumLifetime32Dim64Base8** hash-based signature scheme, which provides: @@ -494,26 +542,30 @@ The system uses the **SIGTopLevelTargetSumLifetime32Dim64Base8** hash-based sign - **Stateful signatures**: Uses hierarchical signature tree structure -**Validator Fields:** -Hash-sig key files are automatically indexed based on the validator index (first validator uses `validator_0_*.json`, second uses `validator_1_*.json`, etc.) +**Validator index:** Files are named by global validator index (`validator_0_*`, `validator_1_*`, …). Each index has **proposer** and **attester** keypairs (see manifest field names `proposer_key_pubkey_hex` / `attester_key_pubkey_hex`). #### Genesis config files **Tool's Docker Image**: `PK_DOCKER_IMAGE="ethpandaops/eth-beacon-genesis:pk910-leanchain"` **Source**: https://github.com/ethpandaops/eth-beacon-genesis/pull/36 -`config.yaml` is generated with the appropriate genesis time (in short future) along with the list pubkeys of the validators in the correct sequence. For e.g: +`config.yaml` is generated with the appropriate genesis time (in the near future), `ATTESTATION_COMMITTEE_COUNT`, and **`GENESIS_VALIDATORS`** as a list of objects with **`attestation_pubkey`** and **`proposal_pubkey`** (104 hex chars each, no `0x` prefix), in validator order. Example: ```yaml # Genesis Settings GENESIS_TIME: 1763712794 +# Chain Settings +ATTESTATION_COMMITTEE_COUNT: 1 # Key Settings ACTIVE_EPOCH: 10 # Validator Settings VALIDATOR_COUNT: 2 +# List of Genesis Validators' Public Keys (attestation + proposal) GENESIS_VALIDATORS: - - "4b3c31094bcc9b45446b2028eae5ad192b2df16778837b10230af102255c9c5f72d7ba43eae30b2c6a779f47367ebf5a42f6c959" - - "8df32a54d2fbdf3a88035b2fe3931320cb900d364d6e7c56b19c0f3c6006ce5b3ebe802a65fe1b420183f62e830a953cb33b7804" + - attestation_pubkey: "4b3c31094bcc9b45446b2028eae5ad192b2df16778837b10230af102255c9c5f72d7ba43eae30b2c6a779f47367ebf5a42f6c959" + proposal_pubkey: "8df32a54d2fbdf3a88035b2fe3931320cb900d364d6e7c56b19c0f3c6006ce5b3ebe802a65fe1b420183f62e830a953cb33b7804" + - attestation_pubkey: "5b15f72f90bd655b039f9839c36951454b89c605f8c334581cfa832bdd0c994a1350094f7e22617d77607b067b0aa2439e0ead7d" + proposal_pubkey: "71bf8f73980591574de34a0db471da74f5cfd84d4731d53f47bf3023b26c2638ac5bd24993ea71492fedbd6c4afe5c299213b76b" ``` This `config.yaml` is consumed by the clients to directly generate the genesis `in-client`. Note that clients are supposed to ignore `genesis.ssz` and `genesis.json` as their formats have not been updated. @@ -531,30 +583,24 @@ qlean_0: - 2 ``` -**Recommended:** `annotated_validators.yaml` is also generated and should be preferred by client software as it includes public keys and private key file references directly, eliminating the need for clients to derive key filenames from validator indices: +**Recommended:** `annotated_validators.yaml` is also generated and should be preferred by client software as it includes public keys and private key file references directly, eliminating the need for clients to derive key filenames from validator indices. With the dual-key manifest, each validator index appears **twice** (attester + proposer SSZ keys): ```yaml zeam_0: - index: 0 pubkey_hex: 4b3c31094bcc9b45446b2028eae5ad192b2df16778837b10230af102255c9c5f72d7ba43eae30b2c6a779f47367ebf5a42f6c959 - privkey_file: validator_0_sk.json - - index: 3 + privkey_file: validator_0_attester_key_sk.ssz + - index: 0 pubkey_hex: 8df32a54d2fbdf3a88035b2fe3931320cb900d364d6e7c56b19c0f3c6006ce5b3ebe802a65fe1b420183f62e830a953cb33b7804 - privkey_file: validator_3_sk.json - -ream_0: - - index: 1 - pubkey_hex: 5b15f72f90bd655b039f9839c36951454b89c605f8c334581cfa832bdd0c994a1350094f7e22617d77607b067b0aa2439e0ead7d - privkey_file: validator_1_sk.json - - index: 4 - pubkey_hex: 71bf8f73980591574de34a0db471da74f5cfd84d4731d53f47bf3023b26c2638ac5bd24993ea71492fedbd6c4afe5c299213b76b - privkey_file: validator_4_sk.json - -qlean_0: - - index: 2 - pubkey_hex: b87e69568a347d1aa811cc158634fb1f4e247c5509ad2b1652a8d758ec0ab0796954e307b97dd6284fbb30088c2e595546fdf663 - privkey_file: validator_2_sk.json + privkey_file: validator_0_proposer_key_sk.ssz + - index: 3 + pubkey_hex: ... + privkey_file: validator_3_attester_key_sk.ssz + - index: 3 + pubkey_hex: ... + privkey_file: validator_3_proposer_key_sk.ssz ``` +(Legacy single-row-per-index entries with `validator_N_sk.ssz` may still appear when using an older manifest.) `nodes.yaml` provide enrs of all the nodes so that clients don't have to run a discovery protocol: @@ -568,7 +614,7 @@ qlean_0: Post genesis generation, the quickstarts loads and calls the appropriate node's client cmd from `client-cmds` folder where either `docker` or `binary` cmd is picked as per the `node_setup` mode. (Generally `binary` mode is handy for local interop debugging for a client). **Client Integration:** -Your client implementation should read these environment variables and use the hash-sig keys for validator operations. +Your client implementation should read these environment variables and use the hash-sig keys for validator operations. After `parse-vc.sh` runs, **`$HASH_SIG_PK_PATH` / `$HASH_SIG_SK_PATH`** point at the **proposer** JSON keys when using dual-key manifest files; **`$HASH_SIG_ATTESTER_PK_PATH`** / **`$HASH_SIG_ATTESTER_SK_PATH`** (and proposer-specific `HASH_SIG_PROPOSER_*`) are set when those files exist. - `$item` - the node name for which this cmd is being executed, index into `validator-config.yaml` for its configuration - `$configDir` - the abs folder housing `genesis` configuration (same as `NETWORK_DIR` env variable provided while executing shell command), already mapped to `/config` in the docker mode @@ -595,7 +641,7 @@ node_binary="$scriptDir/qlean/build/src/executable/qlean \ --listen-addr /ip4/0.0.0.0/udp/$quicPort/quic-v1 \ --metrics-port $metricsPort" -node_docker="--platform linux/amd64 qdrvm/qlean-mini:latest \ +node_docker="--platform linux/amd64 qdrvm/qlean-mini:devnet-4-amd64 \ --genesis /config/config.yaml \ --validator-registry-path /config/validators.yaml \ --bootnodes /config/nodes.yaml \ @@ -646,7 +692,7 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --forceKeyG ### Key Security **Secret keys are highly sensitive:** -- ⚠️ **Never commit** `validator_*_sk.json` files to version control +- ⚠️ **Never commit** `validator_*_*_sk.json` or `validator_*_sk.json` secret key files to version control - ⚠️ **Never share** secret keys - ✅ **Backup** secret keys in secure, encrypted storage - ✅ **Restrict permissions** on key files (e.g., `chmod 600`) @@ -674,21 +720,27 @@ num_validators: 2 validators: - index: 0 - pubkey_hex: 0x4b3c31094bcc9b45446b2028eae5ad192b2df16778837b10230af102255c9c5f72d7ba43eae30b2c6a779f47367ebf5a42f6c959 - privkey_file: validator_0_sk.json + attester_key_pubkey_hex: 0x... + attester_key_privkey_file: validator_0_attester_key_sk.ssz + proposer_key_pubkey_hex: 0x... + proposer_key_privkey_file: validator_0_proposer_key_sk.ssz - index: 1 - pubkey_hex: 0x8df32a54d2fbdf3a88035b2fe3931320cb900d364d6e7c56b19c0f3c6006ce5b3ebe802a65fe1b420183f62e830a953cb33b7804 - privkey_file: validator_1_sk.json + attester_key_pubkey_hex: 0x... + attester_key_privkey_file: validator_1_attester_key_sk.ssz + proposer_key_pubkey_hex: 0x... + proposer_key_privkey_file: validator_1_proposer_key_sk.ssz ``` +(See [hash-sig-cli](https://github.com/blockblaz/hash-sig-cli) for the exact manifest schema.) ## Troubleshooting **Problem**: Hash-sig keys not loading during node startup ``` -Warning: Hash-sig public key not found at genesis/hash-sig-keys/validator_0_pk.json +Warning: Hash-sig public key not found at genesis/hash-sig-keys/validator_0_proposer_key_pk.json ``` +(or `validator_0_pk.json` when using a legacy single-key tree) **Solution**: Run the genesis generator to create keys: ```sh @@ -703,8 +755,9 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis **Problem**: Hash-sig key file not found ``` -Warning: Hash-sig secret key not found at genesis/hash-sig-keys/validator_5_sk.json +Warning: Hash-sig secret key not found at genesis/hash-sig-keys/validator_5_proposer_key_sk.json ``` +(or `validator_5_sk.json` in legacy layouts) **Solution**: This usually means you have more validators configured than hash-sig keys generated. Regenerate genesis files: ```sh diff --git a/TESTING_DEVNET3.md b/TESTING_DEVNET3.md index 3b948b2..9ea1831 100644 --- a/TESTING_DEVNET3.md +++ b/TESTING_DEVNET3.md @@ -50,7 +50,7 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node "zeam_0 ream_0" --generateGenesis ### Test with Attestation Committee Count Override ```bash # 1. Uncomment attestation_committee_count in local-devnet/genesis/validator-config.yaml -# 2. Set desired value (e.g., attestation_committee_count: 4) +# 2. Set desired value (default/spec is 1; e.g., attestation_committee_count: 4 for multi-committee tests) # 3. Run: NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --generateGenesis --cleanData ``` @@ -64,15 +64,15 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --generateGenesis --cleanD 4. zeam command includes `--is-aggregator` flag for aggregator only ### Attestation Committee Count -**When NOT set (default):** +**When NOT set in validator-config.yaml:** - parse-vc.sh does NOT display "Attestation Committee Count" - zeam command does NOT include `--attestation-committee-count` flag -- Client uses its hardcoded default +- zeam reads `ATTESTATION_COMMITTEE_COUNT` from `config.yaml` if present; otherwise chain default **1** (matches leanSpec) -**When set (e.g., to 4):** +**When set in validator-config (e.g., to 4):** - parse-vc.sh displays: "Attestation Committee Count: 4" - zeam command includes: `--attestation-committee-count 4` -- Client uses the specified value +- Client uses the specified value (overrides `config.yaml` when passed) ## Verification diff --git a/ansible-devnet/genesis/test-validator-config-subnet2.yaml b/ansible-devnet/genesis/test-validator-config-subnet2.yaml new file mode 100644 index 0000000..168e828 --- /dev/null +++ b/ansible-devnet/genesis/test-validator-config-subnet2.yaml @@ -0,0 +1,161 @@ +shuffle: roundrobin +# Test layout: 2 subnets × 7 nodes each (1 zeam + 1 grandine + 1 gean + 2 ethlambda + 1 qlean + 1 lantern per subnet +# except subnet 1 which has 1 ream instead of ethlambda_3). +# Each server hosts one node per subnet per client type on distinct ports. +# +# Server layout (IP → nodes): +# 37.27.89.135 grandine_0 (s0,9001) zeam_3 (s1,9002) qlean_0 (s0,9003) +# 157.180.20.55 zeam_1 (s0,9001) zeam_4 (s1,9002) qlean_1 (s1,9003) +# 178.104.133.162 gean_0 (s0,9001) ethlambda_2 (s1,9002) +# 178.104.151.50 ethlambda_0 (s0,9001) ream_0 (s1,9002) lantern_0 (s0,9003) +# 178.104.149.91 ethlambda_1 (s0,9001) gean_1 (s1,9002) lantern_1 (s1,9003) +# +# ./spin-node.sh --validatorConfig ansible-devnet/genesis/test-validator-config-subnet2.yaml +deployment_mode: ansible +config: + activeEpoch: 18 + keyType: "hash-sig" + attestation_committee_count: 2 +validators: + # ── Subnet 0 ────────────────────────────────────────────────────────────── + - name: "grandine_0" + privkey: "c05937b251889e35c58d4601c29bed8153dc22c548448f85e0ab9ca436d4b904" + enrFields: + ip: "37.27.89.135" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + subnet: 0 + isAggregator: false + count: 1 + - name: "zeam_1" + privkey: "e7904333a18df63252e2c807f65915e1a256667ac8af8fb2116186cae2b24d98" + enrFields: + ip: "157.180.20.55" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + subnet: 0 + isAggregator: true + count: 1 + - name: "gean_0" + privkey: "69c251cdb06039dd99d87e5a1439fa3720615be98c293ec9bcfd041877a2e8ca" + enrFields: + ip: "178.104.133.162" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + subnet: 0 + isAggregator: false + count: 1 + - name: "ethlambda_0" + privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" + enrFields: + ip: "178.104.151.50" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + subnet: 0 + isAggregator: false + count: 1 + - name: "ethlambda_1" + privkey: "58c22d2c5785e9a47004f429083804dfc5dad666b4ab6c38face9993fd644c8e" + enrFields: + ip: "178.104.149.91" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + subnet: 0 + isAggregator: false + count: 1 + - name: "qlean_0" + privkey: "8e9f81c9caa9e29d26a7327311ca63a38254efdfccf3ce1362bae47eae0b18b3" + enrFields: + ip: "37.27.89.135" + quic: 9003 + metricsPort: 9097 + apiPort: 5057 + subnet: 0 + isAggregator: false + count: 1 + - name: "lantern_0" + privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" + enrFields: + ip: "178.104.151.50" + quic: 9003 + metricsPort: 9097 + apiPort: 5057 + subnet: 0 + isAggregator: false + count: 1 + # ── Subnet 1 ────────────────────────────────────────────────────────────── + - name: "zeam_3" + privkey: "db070d74f757c3e191cc9232a3c64db2af180cf4c19e2ea2f4c757931d7b7074" + enrFields: + ip: "37.27.89.135" + quic: 9002 + metricsPort: 9096 + apiPort: 5056 + subnet: 1 + isAggregator: false + count: 1 + - name: "zeam_4" + privkey: "f2d3b7044eed1f9edaee061f3d5ec3157408bf45a0f4490a70ae611153cebee4" + enrFields: + ip: "157.180.20.55" + quic: 9002 + metricsPort: 9096 + apiPort: 5056 + subnet: 1 + isAggregator: false + count: 1 + - name: "ethlambda_2" + privkey: "c8f682e8c2156f32dd827ca91a0d1be802f80494514c9e5ff2e35576df06da6d" + enrFields: + ip: "178.104.133.162" + quic: 9002 + metricsPort: 9096 + apiPort: 5056 + subnet: 1 + isAggregator: true + count: 1 + - name: "ream_0" + privkey: "04e5fb8c58dccfe9086d2488ed03f69089a74aacc22e41c06a3b83ac40c5eab6" + enrFields: + ip: "178.104.151.50" + quic: 9002 + metricsPort: 9096 + apiPort: 5056 + subnet: 1 + isAggregator: false + count: 1 + - name: "gean_1" + privkey: "310139c3eecbf3bd49fa2be2ff813fa69b330e2d246266a23cb33a9fa58b0a11" + enrFields: + ip: "178.104.149.91" + quic: 9002 + metricsPort: 9096 + apiPort: 5056 + subnet: 1 + isAggregator: false + count: 1 + - name: "qlean_1" + privkey: "1991db904e455114bc836a0860689e77c1d2fa313a15567c6c7aa81e180925ac" + enrFields: + ip: "157.180.20.55" + quic: 9003 + metricsPort: 9097 + apiPort: 5057 + subnet: 1 + isAggregator: false + count: 1 + - name: "lantern_1" + privkey: "582644c8e66b61df3dafb33fa496cee97060566542fb8594fca661be3332710f" + enrFields: + ip: "178.104.149.91" + quic: 9003 + metricsPort: 9097 + apiPort: 5057 + subnet: 1 + isAggregator: false + count: 1 diff --git a/ansible-devnet/genesis/test-validator-config.yaml b/ansible-devnet/genesis/test-validator-config.yaml new file mode 100644 index 0000000..61c85c4 --- /dev/null +++ b/ansible-devnet/genesis/test-validator-config.yaml @@ -0,0 +1,235 @@ +shuffle: roundrobin +# Test layout: three zeam nodes and two ethlambda nodes on a single subnet, +# each node on its own server (unique IP). +# ./spin-node.sh --validatorConfig ansible-devnet/genesis/test-validator-config.yaml +deployment_mode: ansible +config: + activeEpoch: 18 + keyType: "hash-sig" + attestation_committee_count: 1 +validators: + - name: "zeam_0" + privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" + enrFields: + ip: "37.27.89.135" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "zeam_1" + privkey: "e7904333a18df63252e2c807f65915e1a256667ac8af8fb2116186cae2b24d98" + enrFields: + ip: "157.180.20.55" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "zeam_2" + privkey: "11ff3a52375898532680c86739dbe0b632088545d0b901181d92053b5fab8d38" + enrFields: + ip: "178.104.133.162" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + isAggregator: false + count: 1 + # - name: "zeam_3" + # privkey: "877c1489e75914bd46dccb71e0ee2d32af337fbb82f2c9caea73818e00ea9a61" + # enrFields: + # ip: "37.27.89.135" + # quic: 9004 + # metricsPort: 9098 + # apiPort: 5058 + # isAggregator: false + # count: 1 + # - name: "zeam_4" + # privkey: "2aed04fc149298af1376aee0161d5dc67f9d7a47e625a58ae8e44f210d45c2a2" + # enrFields: + # ip: "37.27.89.135" + # quic: 9005 + # metricsPort: 9099 + # apiPort: 5059 + # isAggregator: false + # count: 1 + # - name: "ream_0" + # privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" + # enrFields: + # ip: "157.180.20.55" + # quic: 9001 + # metricsPort: 9095 + # apiPort: 5055 + # isAggregator: false + # count: 1 + # - name: "ream_1" + # privkey: "01ece82f314ab0a8b4271d2f6a873922d3fb17491d0c8f74fc14ca748ca49b1b" + # enrFields: + # ip: "157.180.20.55" + # quic: 9002 + # metricsPort: 9096 + # apiPort: 5056 + # isAggregator: false + # count: 1 + # - name: "ream_2" + # privkey: "d723f0cc94645848bbacf8aea4e01944e2f4a6f9f2c69b17ed1868f492374a7a" + # enrFields: + # ip: "157.180.20.55" + # quic: 9003 + # metricsPort: 9097 + # apiPort: 5057 + # isAggregator: false + # count: 1 + # - name: "ream_3" + # privkey: "4701dd38aa11a1affd69b5d95e535210f8c29e720222b906c46bd64cf018f9c7" + # enrFields: + # ip: "157.180.20.55" + # quic: 9004 + # metricsPort: 9098 + # apiPort: 5058 + # isAggregator: false + # count: 1 + # - name: "ream_4" + # privkey: "62e6c79311ce4ee1b95ff52c8d6b7cccea0bf0b927fbed595d454858b32c49d6" + # enrFields: + # ip: "157.180.20.55" + # quic: 9005 + # metricsPort: 9099 + # apiPort: 5059 + # isAggregator: false + # count: 1 + - name: "ethlambda_0" + privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" + enrFields: + ip: "178.104.151.50" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "ethlambda_1" + privkey: "58c22d2c5785e9a47004f429083804dfc5dad666b4ab6c38face9993fd644c8e" + enrFields: + ip: "178.104.149.91" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + isAggregator: true + count: 1 +# - name: "ethlambda_2" +# privkey: "4a7c5b2077d0e3c6f5322d0f2fd98cb2efffbceb41de2799e520b48872dc4102" +# enrFields: +# ip: "178.104.151.50" +# quic: 9003 +# metricsPort: 9097 +# apiPort: 5057 +# isAggregator: false +# count: 1 +# - name: "ethlambda_3" +# privkey: "1c454b0399cd2178b46e42c8b599e34f2e7ddab71df96b1fbf510d5951335ea0" +# enrFields: +# ip: "178.104.151.50" +# quic: 9004 +# metricsPort: 9098 +# apiPort: 5058 +# isAggregator: false +# count: 1 +# - name: "ethlambda_4" +# privkey: "91c2c932a6589e130bfa5281cc27f6b9930ba7f99aa043d334f14578ef7492f1" +# enrFields: +# ip: "178.104.151.50" +# quic: 9005 +# metricsPort: 9099 +# apiPort: 5059 +# isAggregator: false +# count: 1 +# - name: "qlean_0" +# privkey: "c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9" +# enrFields: +# ip: "178.104.133.162" +# quic: 9001 +# metricsPort: 9095 +# apiPort: 5055 +# isAggregator: false +# count: 1 +# - name: "qlean_1" +# privkey: "927eed3828e0af660ab7525d0971f7f29a3055f8832b947b03eab0bc5683bd97" +# enrFields: +# ip: "178.104.133.162" +# quic: 9002 +# metricsPort: 9096 +# apiPort: 5056 +# isAggregator: false +# count: 1 +# - name: "qlean_2" +# privkey: "7fdceb8166b9d7ab4169d80345ab85fd042b3684ba307c4c02e851c1520975b5" +# enrFields: +# ip: "178.104.133.162" +# quic: 9003 +# metricsPort: 9097 +# apiPort: 5057 +# isAggregator: false +# count: 1 +# - name: "qlean_3" +# privkey: "1027ce52661ef76b5ada68bc8cd45c13ebefb09c99dcad251b9bd4893c71815f" +# enrFields: +# ip: "178.104.133.162" +# quic: 9004 +# metricsPort: 9098 +# apiPort: 5058 +# isAggregator: false +# count: 1 +# - name: "qlean_4" +# privkey: "0763e1a9f1909df7eae31ef4e93d5aaf45a506624ef81df893fc02e08a4ab94f" +# enrFields: +# ip: "178.104.133.162" +# quic: 9005 +# metricsPort: 9099 +# apiPort: 5059 +# isAggregator: true +# count: 1 +# - name: "lantern_0" +# privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" +# enrFields: +# ip: "178.104.149.91" +# quic: 9001 +# metricsPort: 9095 +# apiPort: 5055 +# isAggregator: false +# count: 1 +# - name: "lantern_1" +# privkey: "582644c8e66b61df3dafb33fa496cee97060566542fb8594fca661be3332710f" +# enrFields: +# ip: "178.104.149.91" +# quic: 9002 +# metricsPort: 9096 +# apiPort: 5056 +# isAggregator: false +# count: 1 +# - name: "lantern_2" +# privkey: "943151f2327ea91357632dd8a3a498242b07eeeebad973ea82d6bead80627d3e" +# enrFields: +# ip: "178.104.149.91" +# quic: 9003 +# metricsPort: 9097 +# apiPort: 5057 +# isAggregator: false +# count: 1 +# - name: "lantern_3" +# privkey: "31258c1b84e4a64954fb18dbb04ec9de7079ad4d0273b40380aa1f4ca2f20804" +# enrFields: +# ip: "178.104.149.91" +# quic: 9004 +# metricsPort: 9098 +# apiPort: 5058 +# isAggregator: false +# count: 1 +# - name: "lantern_4" +# privkey: "a5dac14f317084d255c57e5080985b1dbd79cf8fc625a9c7fba781abf07001bd" +# enrFields: +# ip: "178.104.149.91" +# quic: 9005 +# metricsPort: 9099 +# apiPort: 5059 +# isAggregator: true +# count: 1 diff --git a/ansible-devnet/genesis/validator-config-expanded.yaml b/ansible-devnet/genesis/validator-config-expanded.yaml new file mode 100644 index 0000000..c0238c0 --- /dev/null +++ b/ansible-devnet/genesis/validator-config-expanded.yaml @@ -0,0 +1,257 @@ +shuffle: roundrobin +deployment_mode: ansible +config: + activeEpoch: 18 + keyType: hash-sig + attestation_committee_count: 5 +validators: +- name: zeam_0 + privkey: bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5 + enrFields: + ip: 37.27.89.135 + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + isAggregator: false + count: 1 + subnet: 0 +- name: zeam_1 + privkey: e7904333a18df63252e2c807f65915e1a256667ac8af8fb2116186cae2b24d98 + enrFields: + ip: 37.27.89.135 + quic: 9002 + metricsPort: 9096 + apiPort: 5056 + isAggregator: false + count: 1 + subnet: 1 +- name: zeam_2 + privkey: 11ff3a52375898532680c86739dbe0b632088545d0b901181d92053b5fab8d38 + enrFields: + ip: 37.27.89.135 + quic: 9003 + metricsPort: 9097 + apiPort: 5057 + isAggregator: false + count: 1 + subnet: 2 +- name: zeam_3 + privkey: 877c1489e75914bd46dccb71e0ee2d32af337fbb82f2c9caea73818e00ea9a61 + enrFields: + ip: 37.27.89.135 + quic: 9004 + metricsPort: 9098 + apiPort: 5058 + isAggregator: false + count: 1 + subnet: 3 +- name: zeam_4 + privkey: 2aed04fc149298af1376aee0161d5dc67f9d7a47e625a58ae8e44f210d45c2a2 + enrFields: + ip: 37.27.89.135 + quic: 9005 + metricsPort: 9099 + apiPort: 5059 + isAggregator: false + count: 1 + subnet: 4 +- name: ream_0 + privkey: af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32 + enrFields: + ip: 157.180.20.55 + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + isAggregator: false + count: 1 + subnet: 0 +- name: ream_1 + privkey: 01ece82f314ab0a8b4271d2f6a873922d3fb17491d0c8f74fc14ca748ca49b1b + enrFields: + ip: 157.180.20.55 + quic: 9002 + metricsPort: 9096 + apiPort: 5056 + isAggregator: false + count: 1 + subnet: 1 +- name: ream_2 + privkey: d723f0cc94645848bbacf8aea4e01944e2f4a6f9f2c69b17ed1868f492374a7a + enrFields: + ip: 157.180.20.55 + quic: 9003 + metricsPort: 9097 + apiPort: 5057 + isAggregator: false + count: 1 + subnet: 2 +- name: ream_3 + privkey: 4701dd38aa11a1affd69b5d95e535210f8c29e720222b906c46bd64cf018f9c7 + enrFields: + ip: 157.180.20.55 + quic: 9004 + metricsPort: 9098 + apiPort: 5058 + isAggregator: false + count: 1 + subnet: 3 +- name: ream_4 + privkey: 62e6c79311ce4ee1b95ff52c8d6b7cccea0bf0b927fbed595d454858b32c49d6 + enrFields: + ip: 157.180.20.55 + quic: 9005 + metricsPort: 9099 + apiPort: 5059 + isAggregator: false + count: 1 + subnet: 4 +- name: ethlambda_0 + privkey: 299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a + enrFields: + ip: 178.104.151.50 + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + isAggregator: false + count: 1 + subnet: 0 +- name: ethlambda_1 + privkey: 58c22d2c5785e9a47004f429083804dfc5dad666b4ab6c38face9993fd644c8e + enrFields: + ip: 178.104.151.50 + quic: 9002 + metricsPort: 9096 + apiPort: 5056 + isAggregator: false + count: 1 + subnet: 1 +- name: ethlambda_2 + privkey: 4a7c5b2077d0e3c6f5322d0f2fd98cb2efffbceb41de2799e520b48872dc4102 + enrFields: + ip: 178.104.151.50 + quic: 9003 + metricsPort: 9097 + apiPort: 5057 + isAggregator: false + count: 1 + subnet: 2 +- name: ethlambda_3 + privkey: 1c454b0399cd2178b46e42c8b599e34f2e7ddab71df96b1fbf510d5951335ea0 + enrFields: + ip: 178.104.151.50 + quic: 9004 + metricsPort: 9098 + apiPort: 5058 + isAggregator: false + count: 1 + subnet: 3 +- name: ethlambda_4 + privkey: 91c2c932a6589e130bfa5281cc27f6b9930ba7f99aa043d334f14578ef7492f1 + enrFields: + ip: 178.104.151.50 + quic: 9005 + metricsPort: 9099 + apiPort: 5059 + isAggregator: false + count: 1 + subnet: 4 +- name: qlean_0 + privkey: c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9 + enrFields: + ip: 178.104.133.162 + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + isAggregator: false + count: 1 + subnet: 0 +- name: qlean_1 + privkey: 927eed3828e0af660ab7525d0971f7f29a3055f8832b947b03eab0bc5683bd97 + enrFields: + ip: 178.104.133.162 + quic: 9002 + metricsPort: 9096 + apiPort: 5056 + isAggregator: false + count: 1 + subnet: 1 +- name: qlean_2 + privkey: 7fdceb8166b9d7ab4169d80345ab85fd042b3684ba307c4c02e851c1520975b5 + enrFields: + ip: 178.104.133.162 + quic: 9003 + metricsPort: 9097 + apiPort: 5057 + isAggregator: false + count: 1 + subnet: 2 +- name: qlean_3 + privkey: 1027ce52661ef76b5ada68bc8cd45c13ebefb09c99dcad251b9bd4893c71815f + enrFields: + ip: 178.104.133.162 + quic: 9004 + metricsPort: 9098 + apiPort: 5058 + isAggregator: false + count: 1 + subnet: 3 +- name: qlean_4 + privkey: 0763e1a9f1909df7eae31ef4e93d5aaf45a506624ef81df893fc02e08a4ab94f + enrFields: + ip: 178.104.133.162 + quic: 9005 + metricsPort: 9099 + apiPort: 5059 + isAggregator: false + count: 1 + subnet: 4 +- name: lantern_0 + privkey: d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5 + enrFields: + ip: 178.104.149.91 + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + isAggregator: false + count: 1 + subnet: 0 +- name: lantern_1 + privkey: 582644c8e66b61df3dafb33fa496cee97060566542fb8594fca661be3332710f + enrFields: + ip: 178.104.149.91 + quic: 9002 + metricsPort: 9096 + apiPort: 5056 + isAggregator: false + count: 1 + subnet: 1 +- name: lantern_2 + privkey: 943151f2327ea91357632dd8a3a498242b07eeeebad973ea82d6bead80627d3e + enrFields: + ip: 178.104.149.91 + quic: 9003 + metricsPort: 9097 + apiPort: 5057 + isAggregator: false + count: 1 + subnet: 2 +- name: lantern_3 + privkey: 31258c1b84e4a64954fb18dbb04ec9de7079ad4d0273b40380aa1f4ca2f20804 + enrFields: + ip: 178.104.149.91 + quic: 9004 + metricsPort: 9098 + apiPort: 5058 + isAggregator: false + count: 1 + subnet: 3 +- name: lantern_4 + privkey: a5dac14f317084d255c57e5080985b1dbd79cf8fc625a9c7fba781abf07001bd + enrFields: + ip: 178.104.149.91 + quic: 9005 + metricsPort: 9099 + apiPort: 5059 + isAggregator: false + count: 1 + subnet: 4 diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index 8ee1ce8..9084f25 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -1,163 +1,173 @@ shuffle: roundrobin -# Deployment mode: 'local' for local deployment, 'ansible' for remote deployment via Ansible +# devnet-4 layout: 2 subnets × 8 clients each. One of every unique client type +# (zeam, ream, qlean, lantern, grandine, ethlambda, gean, nlean) per subnet. +# One validator per server (default ports 9001 / 9095 / 5055). Aggregator per +# subnet is chosen randomly at launch by spin-node.sh (all isAggregator: false). deployment_mode: ansible config: activeEpoch: 18 keyType: "hash-sig" + attestation_committee_count: 2 validators: + # ── Subnet 0 ────────────────────────────────────────────────────────────── - name: "zeam_0" - # node id 7d0904dc6d8d7130e0e68d5d3175d0c3cf470f8725f67bd8320882f5b9753cc0 - # peer id 16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" enrFields: - # verify /ip4/46.224.123.223/udp/9001/quic-v1/p2p/16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f - ip: "204.168.135.7" + ip: "95.216.154.185" quic: 9001 metricsPort: 9095 apiPort: 5055 - isAggregator: false - count: 1 # number of indices for this node - + subnet: 0 + isAggregator: true + count: 1 - name: "ream_0" - # node id bc531fc1a99a896acb45603f28a32f81ae607480af46435009de4609370cb7bb - # peer id 16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" enrFields: - #verify /ip4/77.42.27.219/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 - ip: "95.216.154.185" + ip: "204.168.135.7" quic: 9001 metricsPort: 9095 apiPort: 5055 + subnet: 0 isAggregator: false - devnet: 1 count: 1 - - name: "qlean_0" - # node id f0af4dcd8864372ca01ae984b9a386f86e5cb582331394db3434fea59ad78bde - # peer id 16Uiu2HAmQj1RDNAxopeeeCFPRr3zhJYmH6DEPHYKmxLViLahWcFE - privkey: "c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9" + privkey: "8e9f81c9caa9e29d26a7327311ca63a38254efdfccf3ce1362bae47eae0b18b3" enrFields: - #verify /ip4/46.224.123.220/udp/9001/quic-v1/p2p/16Uiu2HAmQj1RDNAxopeeeCFPRr3zhJYmH6DEPHYKmxLViLahWcFE ip: "65.21.182.45" quic: 9001 metricsPort: 9095 apiPort: 5055 + subnet: 0 isAggregator: false count: 1 - - name: "lantern_0" - # node id 974c5fd0a680f5e576189123d5808cefd6c748c75effde28c5c3aacbb8d4c652 - # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" - # verify /ip4/46.224.135.177/udp/9001/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: ip: "65.109.131.177" quic: 9001 metricsPort: 9095 apiPort: 5055 + subnet: 0 isAggregator: false count: 1 - - - name: "lighthouse_0" - # node id af7df69e602552d2ba639e09ca87107b1bc315306ed434aa86c8515dcad0a252 - # peer id 16Uiu2HAmTrYwvBDY3GLEv5zR48ZfBwCYyKgmuLTDsCdBJUo127HY - privkey: "4fd22cf461fbeae4947a3fdaef8d533fc7fd1ef1ce4cd98e993210c18234df3f" - # verify /ip4/46.224.135.169/udp/9001/quic-v1/p2p/16Uiu2HAmTrYwvBDY3GLEv5zR48ZfBwCYyKgmuLTDsCdBJUo127HY + - name: "grandine_0" + privkey: "c05937b251889e35c58d4601c29bed8153dc22c548448f85e0ab9ca436d4b904" enrFields: ip: "65.109.138.213" quic: 9001 metricsPort: 9095 apiPort: 5055 + subnet: 0 isAggregator: false count: 1 - - - name: "grandine_0" - # node id f9e3dfd0eb2e8fb7dfb072b91b23e3286c0fae245cd2c732f4752577f32e5782 - # peer id: 16Uiu2HAmErASHzouSQumaetyU18brt4HNUm4151K55ySBQMRLzSS - privkey: "64a7f5ab53907966374ca23af36392910af682eec82c12e3abbb6c2ccdf39a72" - # verify /ip4/37.27.250.20/udp/9001/quic-v1/p2p/16Uiu2HAmErASHzouSQumaetyU18brt4HNUm4151K55ySBQMRLzSS + - name: "ethlambda_0" + privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" + enrFields: + ip: "204.168.134.201" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + subnet: 0 + isAggregator: false + count: 1 + - name: "gean_0" + privkey: "69c251cdb06039dd99d87e5a1439fa3720615be98c293ec9bcfd041877a2e8ca" + enrFields: + ip: "95.217.19.42" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + subnet: 0 + isAggregator: false + count: 1 + - name: "nlean_0" + privkey: "2e9be3f1b0d32ca3a4d62017fbfafe3950b7e90fed6802ff8bd2e0f8c4e2ca91" + enrFields: + ip: "95.216.173.151" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + subnet: 0 + isAggregator: false + count: 1 + # ── Subnet 1 ────────────────────────────────────────────────────────────── + - name: "zeam_1" + privkey: "8d14c7b02d55ca050ef97a3961aa16828837fb363e4e19e4dd0060f58670a2b3" + enrFields: + ip: "95.216.164.165" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + subnet: 1 + isAggregator: false + count: 1 + - name: "ream_1" + privkey: "fc2f11f90dd90df33bfb5f3467c2cbc37cf93dbb41c7ac556f9eea117418e73d" + enrFields: + ip: "95.216.165.186" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + subnet: 1 + isAggregator: false + count: 1 + - name: "qlean_1" + privkey: "0e190e06a62db01bf566af4348277463d26eebab8f6badbcb989242ea4fee050" enrFields: ip: "37.27.250.20" quic: 9001 metricsPort: 9095 apiPort: 5055 + subnet: 1 isAggregator: false count: 1 - - - name: "ethlambda_0" - # node id 86cec93353ffcc6f38c9c15becadcb30d1acab21a92b30418110c980a38d9c3d - # peer id 16Uiu2HAmPV5jU62WtmDkCEmfq1jzbBDkGbHNsDN78gJyvmv2TuC5 - privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" - # verify /ip4/78.47.44.215/udp/9001/quic-v1/p2p/16Uiu2HAmPV5jU62WtmDkCEmfq1jzbBDkGbHNsDN78gJyvmv2TuC5 + - name: "lantern_1" + privkey: "1176609530e568ec12ad227e60ae71c618bfb13fde5a3257e4b0627216e78e04" enrFields: ip: "78.47.44.215" quic: 9001 metricsPort: 9095 apiPort: 5055 + subnet: 1 isAggregator: false count: 1 - - - name: "gean_0" - # node id d8cfb453fbcbcf6bab2fe3d73d6b476101a813e40062e264a7173c79ad51f2b8 - # peer id 16Uiu2HAmGbnwXEdBYDtARoSbkQ6wjtakg8EynnNaPcksatU5acXr - privkey: "df008e968231c25c3938d80fee9bcc93b4b9711312cf471c1b6f77e67ad68d08" + - name: "grandine_1" + privkey: "ab6edee1173379f647b4022d74d4b3342d547e7bb6954664a1a489b95b7c9b60" enrFields: - # verify /ip4/204.168.134.201/udp/9001/quic-v1/p2p/16Uiu2HAmGbnwXEdBYDtARoSbkQ6wjtakg8EynnNaPcksatU5acXr - ip: "204.168.134.201" + ip: "37.27.89.135" quic: 9001 metricsPort: 9095 apiPort: 5055 + subnet: 1 isAggregator: false count: 1 - - - name: "nlean_0" - # node id 120f7715439bfdcb8025d26ade27488646298a4d9ac99f31ccf4f97fcc60fdfa - # peer id 16Uiu2HAmKyxGoN46ruQqSVzYyNUj2FKTWu7RGCLDGEdpzZ67EAde - privkey: "d94e3dc35e320440c891b66bd82d1aaf2079364162815b32c2633ecae009c84c" + - name: "ethlambda_1" + privkey: "eb8533a0d5071d4dbcb0c4fcf9b8ac6edc3d1a260d2bb348fafc5cdb455aa1d4" enrFields: - # verify /ip4/95.216.164.165/udp/9001/quic-v1/p2p/16Uiu2HAmKyxGoN46ruQqSVzYyNUj2FKTWu7RGCLDGEdpzZ67EAde - ip: "95.216.164.165" + ip: "157.180.20.55" quic: 9001 metricsPort: 9095 apiPort: 5055 + subnet: 1 + isAggregator: true + count: 1 + - name: "gean_1" + privkey: "5408a68b960a7932e367b32489498a13c339f87ff8090ea54213524b3d76fcff" + enrFields: + ip: "178.104.151.50" + quic: 9001 + metricsPort: 9095 + apiPort: 5055 + subnet: 1 isAggregator: false count: 1 - - # - name: "lean_node_0" - # # node id 2289c0a43d51d4f50eee6101c580dbe3aba11eb4a9d32ba9993dd9976c5b994d - # # peer id 16Uiu2HAkxUKxdny94t6WujGhVMBvpQJcTBVFGMGnzzkdyzbNLp7f - # privkey: "520940446264167697a35be635041d651e48989ce2bb36698c444f42a2dd4f6c" - # enrFields: - # # verify /ip4/95.217.19.42/udp/9001/quic-v1/p2p/16Uiu2HAkxUKxdny94t6WujGhVMBvpQJcTBVFGMGnzzkdyzbNLp7f - # ip: "95.217.19.42" - # quic: 9001 - # metricsPort: 9095 - # apiPort: 5055 - # isAggregator: false - # count: 1 - - - name: "peam_0" - # node id 23537364cc6e2547f1ed53909e08e933c53aaed1c32a991e78cdcbc8e62c61cb - # peer id 16Uiu2HAmVSWJiHJMtfuLkyr8Du4BxTLg9rpgR32QHu1u3YuhcM3R - privkey: "8c7b62f4c4a35d134649817f44d2839ad15bcfa1fe6b87991f64d6f54f3ef2a1" - enrFields: - # verify /ip4/95.216.173.151/udp/9001/quic-v1/p2p/16Uiu2HAmVSWJiHJMtfuLkyr8Du4BxTLg9rpgR32QHu1u3YuhcM3R - ip: "95.216.173.151" + - name: "nlean_1" + privkey: "74388a2487b943d947c17bf65ca08470f3ad5f045ea0f5aed38a98bbaba1bd49" + enrFields: + ip: "178.104.133.162" quic: 9001 metricsPort: 9095 apiPort: 5055 + subnet: 1 isAggregator: false count: 1 - - # - name: "node_0" - # # node id 4a27002fe8c79b68acf1e8af321fd4f3407492ab626de16174105234b4e726a7 - # # peer id 16Uiu2HAkxthAvokRQU71Secqos58q38TSXBqHHt5UUM5Viu2EuvF - # privkey: "ebf06c49c66d4d08adaed41e901f49fd7439aa851110cf08c9ae54e9322a696e" - # enrFields: - # # verify /ip4/95.216.165.186/udp/9001/quic-v1/p2p/16Uiu2HAkxthAvokRQU71Secqos58q38TSXBqHHt5UUM5Viu2EuvF - # ip: "95.216.165.186" - # quic: 9001 - # metricsPort: 9095 - # apiPort: 5055 - # isAggregator: false - # count: 1 diff --git a/ansible/playbooks/copy-genesis.yml b/ansible/playbooks/copy-genesis.yml index 02ed759..c9f95ed 100644 --- a/ansible/playbooks/copy-genesis.yml +++ b/ansible/playbooks/copy-genesis.yml @@ -176,6 +176,10 @@ mode: '0600' force: yes loop: "{{ node_hash_sig_files | default([]) }}" + retries: 3 + delay: 5 + register: copy_result + until: copy_result is success when: hash_sig_keys_stat.stat.exists and (node_hash_sig_files | default([]) | length > 0) - name: List files on remote genesis directory diff --git a/ansible/playbooks/deploy-nodes.yml b/ansible/playbooks/deploy-nodes.yml index 8f24766..df71b67 100644 --- a/ansible/playbooks/deploy-nodes.yml +++ b/ansible/playbooks/deploy-nodes.yml @@ -195,6 +195,10 @@ mode: '0600' force: yes loop: "{{ node_hash_sig_files | default([]) }}" + retries: 3 + delay: 5 + register: copy_result + until: copy_result is success when: hash_sig_keys_local.stat.exists and (node_hash_sig_files | default([]) | length > 0) tags: - deploy diff --git a/ansible/playbooks/prepare.yml b/ansible/playbooks/prepare.yml index 10d5d3d..b217902 100644 --- a/ansible/playbooks/prepare.yml +++ b/ansible/playbooks/prepare.yml @@ -38,6 +38,9 @@ # ./spin-node.sh --prepare [--sshKey ~/.ssh/id_ed25519] [--useRoot] # # Only runs on remote hosts (localhost is excluded). +# +# run-ansible.sh uses inventory/hosts-prepare.yml (one host per unique enrFields.ip) +# so Docker/apt run once per machine, not once per zeam_0..zeam_4 on the same IP. - name: Prepare remote servers hosts: all:!localhost @@ -188,7 +191,7 @@ # Both are checked and installed if absent before any rules are applied. # # Port lookup uses the active validator config file, which is the - # --subnets-expanded file (e.g. validator-config-subnets-3.yaml) when + # validator-config-expanded.yaml (from --subnets N) when # --subnets N was passed, or validator-config.yaml otherwise. # validator_config_basename is injected by run-ansible.sh. # diff --git a/ansible/roles/ethlambda/defaults/main.yml b/ansible/roles/ethlambda/defaults/main.yml index fc8589d..87206bd 100644 --- a/ansible/roles/ethlambda/defaults/main.yml +++ b/ansible/roles/ethlambda/defaults/main.yml @@ -3,5 +3,5 @@ # Note: These are fallback defaults. Actual values are extracted from client-cmds/ethlambda-cmd.sh # in the tasks/main.yml file. These defaults are used if extraction fails. -ethlambda_docker_image: "ghcr.io/lambdaclass/ethlambda:latest" +ethlambda_docker_image: "ghcr.io/lambdaclass/ethlambda:devnet4" deployment_mode: docker # docker or binary diff --git a/ansible/roles/ethlambda/tasks/main.yml b/ansible/roles/ethlambda/tasks/main.yml index 35b0fb6..75f2ac4 100644 --- a/ansible/roles/ethlambda/tasks/main.yml +++ b/ansible/roles/ethlambda/tasks/main.yml @@ -26,7 +26,7 @@ - name: Set docker image and deployment mode from client-cmd.sh set_fact: - ethlambda_docker_image: "{{ ethlambda_docker_image_raw.stdout | trim | default('ghcr.io/lambdaclass/ethlambda:latest') }}" + ethlambda_docker_image: "{{ ethlambda_docker_image_raw.stdout | trim | default('ghcr.io/lambdaclass/ethlambda:devnet4') }}" deployment_mode: "{{ ethlambda_deployment_mode_raw.stdout | trim | default('docker') }}" - name: Extract node configuration from validator-config.yaml @@ -50,6 +50,22 @@ ethlambda_is_aggregator: "{{ 'true' if (ethlambda_node_config.results[3].stdout | default('') | trim) == 'true' else 'false' }}" when: ethlambda_node_config is defined +# Compute the full set of subnet ids in the network so aggregators can subscribe +# to attestations from every subnet (not just the one their validators live in). +# Required in multi-subnet deployments for cross-subnet attestation aggregation. +- name: Compute all subnet ids from validator-config.yaml + shell: | + yq eval '[.validators[].subnet] | unique | sort | join(",")' "{{ local_validator_config_path }}" + register: ethlambda_all_subnets_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Set aggregate subnet ids csv + set_fact: + ethlambda_aggregate_subnet_ids: "{{ ethlambda_all_subnets_raw.stdout | trim }}" + run_once: true + - name: Ensure node key file exists stat: path: "{{ genesis_dir }}/{{ node_name }}.key" @@ -104,6 +120,7 @@ --api-port {{ ethlambda_api_port }} --metrics-port {{ ethlambda_metrics_port }} {{ '--is-aggregator' if (ethlambda_is_aggregator | default('false')) == 'true' else '' }} + {{ ('--aggregate-subnet-ids ' + ethlambda_aggregate_subnet_ids) if (ethlambda_is_aggregator | default('false')) == 'true' and (',' in ethlambda_aggregate_subnet_ids | default('')) else '' }} {{ ('--checkpoint-sync-url ' + checkpoint_sync_url) if (checkpoint_sync_url is defined and checkpoint_sync_url | length > 0) else '' }} register: ethlambda_container changed_when: ethlambda_container.rc == 0 diff --git a/ansible/roles/gean/defaults/main.yml b/ansible/roles/gean/defaults/main.yml index 3e9959f..237f136 100644 --- a/ansible/roles/gean/defaults/main.yml +++ b/ansible/roles/gean/defaults/main.yml @@ -3,6 +3,6 @@ # Note: These are fallback defaults. Actual values are extracted from client-cmds/gean-cmd.sh # in the tasks/main.yml file. These defaults are used if extraction fails. -gean_docker_image: "ghcr.io/geanlabs/gean:devnet3" +gean_docker_image: "ghcr.io/geanlabs/gean:devnet4" deployment_mode: docker # docker or binary gean_devnet_id: "devnet0" diff --git a/ansible/roles/gean/tasks/main.yml b/ansible/roles/gean/tasks/main.yml index 83fa44d..8ef1617 100644 --- a/ansible/roles/gean/tasks/main.yml +++ b/ansible/roles/gean/tasks/main.yml @@ -22,7 +22,7 @@ - name: Set docker image and deployment mode from client-cmd.sh set_fact: - gean_docker_image: "{{ gean_docker_image_raw.stdout | trim | default('ghcr.io/geanlabs/gean:devnet3') }}" + gean_docker_image: "{{ gean_docker_image_raw.stdout | trim | default('ghcr.io/geanlabs/gean:devnet4') }}" deployment_mode: "{{ gean_deployment_mode_raw.stdout | trim | default('docker') }}" - name: Extract node configuration from validator-config.yaml @@ -92,17 +92,14 @@ -v {{ data_dir }}/{{ node_name }}:/data {{ gean_docker_image }} --data-dir /data - --genesis /config/config.yaml - --bootnodes /config/nodes.yaml - --validator-registry-path /config/validators.yaml + --custom-network-config-dir /config + --gossipsub-port {{ gean_quic_port }} --node-id {{ node_name }} --node-key /config/{{ node_name }}.key - --validator-keys /config/hash-sig-keys - --listen-addr /ip4/0.0.0.0/udp/{{ gean_quic_port }}/quic-v1 - --discovery-port {{ gean_quic_port }} - --devnet-id {{ gean_devnet_id }} + --http-address 0.0.0.0 --api-port {{ gean_api_port }} --metrics-port {{ gean_metrics_port }} + {{ ('--attestation-committee-count ' + (attestation_committee_count | string)) if (attestation_committee_count is defined and attestation_committee_count | int > 1) else '' }} {{ '--is-aggregator' if (gean_is_aggregator | default('false')) == 'true' else '' }} {{ ('--checkpoint-sync-url ' + checkpoint_sync_url) if (checkpoint_sync_url is defined and checkpoint_sync_url | length > 0) else '' }} register: gean_container diff --git a/ansible/roles/genesis/tasks/main.yml b/ansible/roles/genesis/tasks/main.yml index 88f11ea..e5882df 100644 --- a/ansible/roles/genesis/tasks/main.yml +++ b/ansible/roles/genesis/tasks/main.yml @@ -39,6 +39,8 @@ content: | # Genesis Settings GENESIS_TIME: {{ genesis_time }} + # Chain Settings + ATTESTATION_COMMITTEE_COUNT: 1 # Validator Settings VALIDATOR_COUNT: {{ total_validators }} dest: "{{ genesis_dir }}/config.yaml" diff --git a/ansible/roles/grandine/defaults/main.yml b/ansible/roles/grandine/defaults/main.yml index 6cc516a..03847f2 100644 --- a/ansible/roles/grandine/defaults/main.yml +++ b/ansible/roles/grandine/defaults/main.yml @@ -3,6 +3,6 @@ # Note: These are fallback defaults. Actual values are extracted from client-cmds/grandine-cmd.sh # in the tasks/main.yml file. These defaults are used if extraction fails. -grandine_docker_image: "sifrai/lean:latest" +grandine_docker_image: "sifrai/lean:devnet-4" deployment_mode: docker # docker or binary diff --git a/ansible/roles/grandine/tasks/main.yml b/ansible/roles/grandine/tasks/main.yml index b1018c7..4941385 100644 --- a/ansible/roles/grandine/tasks/main.yml +++ b/ansible/roles/grandine/tasks/main.yml @@ -26,7 +26,7 @@ - name: Set docker image and deployment mode from client-cmd.sh set_fact: - grandine_docker_image: "{{ grandine_docker_image_raw.stdout | trim | default('sifrai/lean:latest') }}" + grandine_docker_image: "{{ grandine_docker_image_raw.stdout | trim | default('sifrai/lean:devnet-4') }}" deployment_mode: "{{ grandine_deployment_mode_raw.stdout | trim | default('docker') }}" - name: Extract node configuration from validator-config.yaml @@ -98,7 +98,7 @@ -v {{ data_dir }}/{{ node_name }}:/data {{ grandine_docker_image }} --genesis /config/config.yaml - --validator-registry-path /config/validators.yaml + --validator-registry-path /config/annotated_validators.yaml --bootnodes /config/nodes.yaml --node-id {{ node_name }} --node-key /config/{{ node_name }}.key diff --git a/ansible/roles/lantern/defaults/main.yml b/ansible/roles/lantern/defaults/main.yml index 64916f2..8bd8a06 100644 --- a/ansible/roles/lantern/defaults/main.yml +++ b/ansible/roles/lantern/defaults/main.yml @@ -3,6 +3,6 @@ # Note: These are fallback defaults. Actual values are extracted from client-cmds/lantern-cmd.sh # in the tasks/main.yml file. These defaults are used if extraction fails. -lantern_docker_image: "piertwo/lantern:v0.0.3" +lantern_docker_image: "piertwo/lantern:v0.0.4" deployment_mode: docker # docker or binary diff --git a/ansible/roles/lantern/tasks/main.yml b/ansible/roles/lantern/tasks/main.yml index 66b77b2..292f395 100644 --- a/ansible/roles/lantern/tasks/main.yml +++ b/ansible/roles/lantern/tasks/main.yml @@ -4,7 +4,7 @@ - name: Set lantern docker image set_fact: - lantern_docker_image: "piertwo/lantern:v0.0.3" + lantern_docker_image: "piertwo/lantern:v0.0.4" - name: Extract deployment mode from client-cmd.sh shell: | @@ -85,6 +85,7 @@ --data-dir /data --genesis-config /config/config.yaml --validator-registry-path /config/validators.yaml + --validator-keys-path /config/annotated_validators.yaml --genesis-state /config/genesis.ssz --validator-config /config/validator-config.yaml --nodes-path /config/nodes.yaml diff --git a/ansible/roles/nlean/defaults/main.yml b/ansible/roles/nlean/defaults/main.yml index 9f1c324..bdf9323 100644 --- a/ansible/roles/nlean/defaults/main.yml +++ b/ansible/roles/nlean/defaults/main.yml @@ -3,6 +3,6 @@ # Note: These are fallback defaults. Actual values are extracted from client-cmds/nlean-cmd.sh # in the tasks/main.yml file. These defaults are used if extraction fails. -nlean_docker_image: "ghcr.io/nleaneth/nlean:latest" +nlean_docker_image: "ghcr.io/nleaneth/nlean:devnet4" deployment_mode: docker # docker or binary nlean_network_name: "devnet0" diff --git a/ansible/roles/nlean/tasks/main.yml b/ansible/roles/nlean/tasks/main.yml index 03b578c..6dbd41b 100644 --- a/ansible/roles/nlean/tasks/main.yml +++ b/ansible/roles/nlean/tasks/main.yml @@ -22,7 +22,7 @@ - name: Set docker image and deployment mode from client-cmd.sh set_fact: - nlean_docker_image: "{{ nlean_docker_image_raw.stdout | trim | default('ghcr.io/nleaneth/nlean:latest') }}" + nlean_docker_image: "{{ nlean_docker_image_raw.stdout | trim | default('ghcr.io/nleaneth/nlean:devnet4') }}" deployment_mode: "{{ nlean_deployment_mode_raw.stdout | trim | default('docker') }}" - name: Extract node configuration from validator-config.yaml @@ -91,7 +91,7 @@ -v {{ genesis_dir }}:/config:ro -v {{ data_dir }}/{{ node_name }}:/data {{ nlean_docker_image }} - --validator-config /config/validator-config.yaml + --custom-network-config-dir /config --node {{ node_name }} --data-dir /data --network {{ nlean_network_name }} @@ -102,8 +102,10 @@ --metrics-address 0.0.0.0 --hash-sig-key-dir /config/hash-sig-keys --api-port {{ nlean_api_port }} + {{ ('--attestation-committee-count ' + (attestation_committee_count | string)) if (attestation_committee_count is defined and attestation_committee_count | int > 1) else '' }} {{ '--is-aggregator' if (nlean_is_aggregator | default('false')) == 'true' else '' }} {{ ('--checkpoint-sync-url ' + checkpoint_sync_url) if (checkpoint_sync_url is defined and checkpoint_sync_url | length > 0) else '' }} + {{ ('--log ' + nlean_log_level) if (nlean_log_level is defined and nlean_log_level | length > 0) else '' }} register: nlean_container changed_when: nlean_container.rc == 0 when: deployment_mode == 'docker' diff --git a/ansible/roles/peam/defaults/main.yml b/ansible/roles/peam/defaults/main.yml index 2fcc884..4b56e61 100644 --- a/ansible/roles/peam/defaults/main.yml +++ b/ansible/roles/peam/defaults/main.yml @@ -3,6 +3,5 @@ # Note: These are fallback defaults. Actual values are extracted from client-cmds/peam-cmd.sh # in the tasks/main.yml file. These defaults are used if extraction fails. -peam_docker_image: "ghcr.io/malik672/peam:devnet3" -peam_devnet_id: "devnet0" +peam_docker_image: "ghcr.io/malik672/peam:devnet4" deployment_mode: docker # docker or binary diff --git a/ansible/roles/peam/tasks/main.yml b/ansible/roles/peam/tasks/main.yml index 4d75ee5..e45d6fe 100644 --- a/ansible/roles/peam/tasks/main.yml +++ b/ansible/roles/peam/tasks/main.yml @@ -22,7 +22,7 @@ - name: Set docker image and deployment mode from client-cmd.sh set_fact: - peam_docker_image: "{{ peam_docker_image_raw.stdout | trim | default('ghcr.io/malik672/peam:devnet3') }}" + peam_docker_image: "{{ peam_docker_image_raw.stdout | trim | default('ghcr.io/malik672/peam:devnet4') }}" deployment_mode: "{{ peam_deployment_mode_raw.stdout | trim | default('docker') }}" - name: Extract node configuration from validator-config.yaml diff --git a/ansible/roles/qlean/defaults/main.yml b/ansible/roles/qlean/defaults/main.yml index 6c02743..48c5ebc 100644 --- a/ansible/roles/qlean/defaults/main.yml +++ b/ansible/roles/qlean/defaults/main.yml @@ -3,9 +3,8 @@ # Note: These are fallback defaults. Actual values are extracted from client-cmds/qlean-cmd.sh # in the tasks/main.yml file. These defaults are used if extraction fails. -qlean_docker_image: "qdrvm/qlean-mini:latest" +qlean_docker_image: "qdrvm/qlean-mini:devnet-4-amd64" qlean_docker_platform: "linux/amd64" -qlean_binary_path: "{{ playbook_dir }}/../qlean/build/src/executable/qlean" -qlean_modules_dir: "{{ playbook_dir }}/../qlean/build/src/modules" +qlean_binary_path: "{{ playbook_dir }}/../qlean/build/out/bin/qlean" deployment_mode: docker # docker or binary diff --git a/ansible/roles/qlean/tasks/main.yml b/ansible/roles/qlean/tasks/main.yml index 651a4a0..bfb5995 100644 --- a/ansible/roles/qlean/tasks/main.yml +++ b/ansible/roles/qlean/tasks/main.yml @@ -24,7 +24,7 @@ - name: Set docker image and deployment mode from client-cmd.sh set_fact: - qlean_docker_image: "{{ qlean_docker_image_raw.stdout | trim | default('qdrvm/qlean-mini:latest') }}" + qlean_docker_image: "{{ qlean_docker_image_raw.stdout | trim | default('qdrvm/qlean-mini:devnet-4-amd64') }}" deployment_mode: "{{ qlean_deployment_mode_raw.stdout | trim | default('docker') }}" - name: Extract node configuration from validator-config.yaml @@ -50,19 +50,6 @@ qlean_is_aggregator: "{{ 'true' if (qlean_node_config.results[4].stdout | default('') | trim) == 'true' else 'false' }}" when: qlean_node_config is defined -- name: Extract validator index from validators.yaml - shell: | - yq eval '."{{ node_name }}" | .[0]' "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validators.yaml" - register: qlean_validator_index - changed_when: false - failed_when: false - delegate_to: localhost - -- name: Set validator index - set_fact: - qlean_validator_index: "{{ qlean_validator_index.stdout | trim | default('0') }}" - when: qlean_validator_index is defined - - name: Ensure node key file exists stat: path: "{{ genesis_dir }}/{{ node_name }}.key" @@ -103,12 +90,7 @@ -v {{ genesis_dir }}:/config:ro -v {{ data_dir }}/{{ node_name }}:/data {{ qlean_docker_image }} - --genesis /config/config.yaml - --validator-registry-path /config/validators.yaml - --validator-keys-manifest /config/hash-sig-keys/validator-keys-manifest.yaml - --xmss-pk /config/hash-sig-keys/validator_{{ qlean_validator_index }}_pk.json - --xmss-sk /config/hash-sig-keys/validator_{{ qlean_validator_index }}_sk.json - --bootnodes /config/nodes.yaml + --genesis-dir /config --data-dir /data --node-id {{ node_name }} --node-key /config/{{ node_name }}.key @@ -120,7 +102,6 @@ {{ '--is-aggregator' if (qlean_is_aggregator | default('false')) == 'true' else '' }} {{ ('--checkpoint-sync-url ' + checkpoint_sync_url) if (checkpoint_sync_url is defined and checkpoint_sync_url | length > 0) else '' }} -ldebug - -ltrace register: qlean_container changed_when: qlean_container.rc == 0 when: deployment_mode == 'docker' diff --git a/ansible/roles/ream/defaults/main.yml b/ansible/roles/ream/defaults/main.yml index cc03dd7..2104885 100644 --- a/ansible/roles/ream/defaults/main.yml +++ b/ansible/roles/ream/defaults/main.yml @@ -3,6 +3,6 @@ # Note: These are fallback defaults. Actual values are extracted from client-cmds/ream-cmd.sh # in the tasks/main.yml file. These defaults are used if extraction fails. -ream_docker_image: "ghcr.io/reamlabs/ream:latest" +ream_docker_image: "ghcr.io/reamlabs/ream:latest-devnet4" deployment_mode: docker # docker or binary diff --git a/ansible/roles/ream/tasks/main.yml b/ansible/roles/ream/tasks/main.yml index f088275..c01cb93 100644 --- a/ansible/roles/ream/tasks/main.yml +++ b/ansible/roles/ream/tasks/main.yml @@ -25,7 +25,7 @@ - name: Set docker image and deployment mode from client-cmd.sh set_fact: - ream_docker_image: "{{ ream_docker_image_raw.stdout | trim | default('ghcr.io/reamlabs/ream:latest') }}" + ream_docker_image: "{{ ream_docker_image_raw.stdout | trim | default('ghcr.io/reamlabs/ream:latest-devnet4') }}" deployment_mode: "{{ ream_deployment_mode_raw.stdout | trim | default('docker') }}" - name: Extract node configuration from validator-config.yaml @@ -93,7 +93,7 @@ --data-dir /data lean_node --network /config/config.yaml - --validator-registry-path /config/validators.yaml + --validator-registry-path /config/annotated_validators.yaml --bootnodes /config/nodes.yaml --node-id {{ node_name }} --node-key /config/{{ node_name }}.key diff --git a/ansible/roles/zeam/defaults/main.yml b/ansible/roles/zeam/defaults/main.yml index 7959f35..cc83b86 100644 --- a/ansible/roles/zeam/defaults/main.yml +++ b/ansible/roles/zeam/defaults/main.yml @@ -3,11 +3,11 @@ # Note: These are fallback defaults. Actual values are extracted from client-cmds/zeam-cmd.sh # in the tasks/main.yml file. These defaults are used if extraction fails. -zeam_docker_image: "blockblaz/zeam:devnet3" +zeam_docker_image: "blockblaz/zeam:devnet4" zeam_binary_path: "{{ playbook_dir }}/../zig-out/bin/zeam" deployment_mode: docker # docker or binary -# Global zeam CLI flags before `node`. Empty by default: published devnet3 image lacks these options. +# Global zeam CLI flags before `node`. Empty by default: published devnet4 image lacks these options. # Override when using a zeam build that supports them, e.g. "--console-log-level debug" zeam_global_flags: "" diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index a7bd04e..1e78bd2 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -31,7 +31,7 @@ - name: Set docker image and deployment mode from client-cmd.sh set_fact: - zeam_docker_image: "{{ zeam_docker_image_raw.stdout | trim | default('blockblaz/zeam:latest') }}" + zeam_docker_image: "{{ zeam_docker_image_raw.stdout | trim | default('blockblaz/zeam:devnet4') }}" deployment_mode: "{{ zeam_deployment_mode_raw.stdout | trim | default('docker') }}" delegate_to: localhost run_once: true @@ -56,6 +56,22 @@ zeam_is_aggregator: "{{ 'true' if (node_config.results[3].stdout | default('') | trim) == 'true' else 'false' }}" when: node_config is defined +# Compute the full set of subnet ids in the network so aggregators can subscribe +# to attestations from every subnet (not just the one their validators live in). +# Required in multi-subnet deployments for cross-subnet attestation aggregation. +- name: Compute all subnet ids from validator-config.yaml + shell: | + yq eval '[.validators[].subnet] | unique | sort | join(",")' "{{ local_validator_config_path }}" + register: zeam_all_subnets_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Set aggregate subnet ids csv + set_fact: + zeam_aggregate_subnet_ids: "{{ zeam_all_subnets_raw.stdout | trim }}" + run_once: true + - name: Ensure node key file exists stat: path: "{{ genesis_dir }}/{{ node_name }}.key" @@ -85,12 +101,9 @@ changed_when: docker_stop_result.rc == 0 - name: Start Zeam container - # TODO: Remove --platform linux/amd64 when blockblaz/zeam:latest multi-platform image is available on Docker Hub - # Multi-platform support is being added in zeam CI (see .github/workflows/ci.yml docker-build-multiarch job) command: >- docker run -d --pull=always - --platform linux/amd64 --name {{ node_name }} --restart unless-stopped --network host @@ -110,6 +123,7 @@ --api-port {{ zeam_api_port }} --metrics-port {{ zeam_metrics_port }} {{ '--is-aggregator' if (zeam_is_aggregator | default('false')) == 'true' else '' }} + {{ ('--aggregate-subnet-ids ' + zeam_aggregate_subnet_ids) if (zeam_is_aggregator | default('false')) == 'true' and (',' in zeam_aggregate_subnet_ids | default('')) else '' }} {{ ('--checkpoint-sync-url ' + checkpoint_sync_url) if (checkpoint_sync_url is defined and checkpoint_sync_url | length > 0) else '' }} register: zeam_container_result changed_when: zeam_container_result.rc == 0 diff --git a/client-cmds/ethlambda-cmd.sh b/client-cmds/ethlambda-cmd.sh index 84e275f..e2b2ab2 100644 --- a/client-cmds/ethlambda-cmd.sh +++ b/client-cmds/ethlambda-cmd.sh @@ -10,6 +10,15 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi +# In multi-subnet deployments, an aggregator must subscribe to every subnet's +# attestation topics so it can aggregate votes from all committees. The caller +# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the +# full subnet id set for the network. +aggregate_subnet_ids_flag="" +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then + aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" +fi + # Set attestation committee count flag if explicitly configured attestation_committee_flag="" if [ -n "$attestationCommitteeCount" ]; then @@ -34,10 +43,11 @@ node_binary="$binary_path \ --metrics-port $metricsPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" # Command when running as docker container -node_docker="ghcr.io/lambdaclass/ethlambda:devnet3 \ +node_docker="ghcr.io/lambdaclass/ethlambda:devnet4 \ --custom-network-config-dir /config \ --data-dir /data \ --gossipsub-port $quicPort \ @@ -48,6 +58,7 @@ node_docker="ghcr.io/lambdaclass/ethlambda:devnet3 \ --metrics-port $metricsPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" node_setup="docker" diff --git a/client-cmds/gean-cmd.sh b/client-cmds/gean-cmd.sh index 0b182eb..53af74f 100644 --- a/client-cmds/gean-cmd.sh +++ b/client-cmds/gean-cmd.sh @@ -9,6 +9,15 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi +# In multi-subnet deployments, an aggregator must subscribe to every subnet's +# attestation topics so it can aggregate votes from all committees. The caller +# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the +# full subnet id set for the network. +aggregate_subnet_ids_flag="" +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then + aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" +fi + # Set attestation committee count flag if explicitly configured attestation_committee_flag="" if [ -n "$attestationCommitteeCount" ]; then @@ -32,10 +41,11 @@ node_binary="$binary_path \ --metrics-port $metricsPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" # Command when running as docker container -node_docker="ghcr.io/geanlabs/gean:devnet3 \ +node_docker="ghcr.io/geanlabs/gean:devnet4 \ --custom-network-config-dir /config \ --gossipsub-port $quicPort \ --node-id $item \ @@ -45,6 +55,7 @@ node_docker="ghcr.io/geanlabs/gean:devnet3 \ --metrics-port $metricsPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" -node_setup="docker" +node_setup="docker" \ No newline at end of file diff --git a/client-cmds/grandine-cmd.sh b/client-cmds/grandine-cmd.sh index 378a827..8828ed6 100644 --- a/client-cmds/grandine-cmd.sh +++ b/client-cmds/grandine-cmd.sh @@ -6,6 +6,15 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi +# In multi-subnet deployments, an aggregator must subscribe to every subnet's +# attestation topics so it can aggregate votes from all committees. The caller +# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the +# full subnet id set for the network. +aggregate_subnet_ids_flag="" +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then + aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" +fi + # Set attestation committee count flag if explicitly configured attestation_committee_flag="" if [ -n "$attestationCommitteeCount" ]; then @@ -20,7 +29,7 @@ fi node_binary="$grandine_bin \ --genesis $configDir/config.yaml \ - --validator-registry-path $configDir/validators.yaml \ + --validator-registry-path $configDir/annotated_validators.yaml \ --bootnodes $configDir/nodes.yaml \ --node-id $item \ --node-key $configDir/$privKeyPath \ @@ -34,11 +43,12 @@ node_binary="$grandine_bin \ --hash-sig-key-dir $configDir/hash-sig-keys \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" -node_docker="sifrai/lean:devnet-3 \ +node_docker="sifrai/lean:devnet-4 \ --genesis /config/config.yaml \ - --validator-registry-path /config/validators.yaml \ + --validator-registry-path /config/annotated_validators.yaml \ --bootnodes /config/nodes.yaml \ --node-id $item \ --node-key /config/$privKeyPath \ @@ -52,6 +62,7 @@ node_docker="sifrai/lean:devnet-3 \ --hash-sig-key-dir /config/hash-sig-keys \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" # choose either binary or docker diff --git a/client-cmds/lantern-cmd.sh b/client-cmds/lantern-cmd.sh index 65da26e..19f1b86 100755 --- a/client-cmds/lantern-cmd.sh +++ b/client-cmds/lantern-cmd.sh @@ -1,7 +1,7 @@ #!/bin/bash #-----------------------lantern setup---------------------- -LANTERN_IMAGE="piertwo/lantern:v0.0.3" +LANTERN_IMAGE="piertwo/lantern:v0.0.4" devnet_flag="" if [ -n "$devnet" ]; then @@ -14,6 +14,15 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi +# In multi-subnet deployments, an aggregator must subscribe to every subnet's +# attestation topics so it can aggregate votes from all committees. The caller +# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the +# full subnet id set for the network. +aggregate_subnet_ids_flag="" +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then + aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" +fi + # Set attestation committee count flag if explicitly configured attestation_committee_flag="" if [ -n "$attestationCommitteeCount" ]; then @@ -26,12 +35,6 @@ if [ -n "${checkpoint_sync_url:-}" ]; then checkpoint_sync_flag="--checkpoint-sync-url $checkpoint_sync_url" fi -# Set attestation committee count flag if explicitly configured -attestation_committee_flag="" -if [ -n "$attestationCommitteeCount" ]; then - attestation_committee_flag="--attestation-committee-count $attestationCommitteeCount" -fi - # Set HTTP port (default to 5055 if not specified in validator-config.yaml) if [ -z "$httpPort" ]; then httpPort="5055" @@ -42,6 +45,7 @@ node_binary="$scriptDir/lantern/build/lantern_cli \ --data-dir $dataDir/$item \ --genesis-config $configDir/config.yaml \ --validator-registry-path $configDir/validators.yaml \ + --validator-keys-path $configDir/annotated_validators.yaml \ --genesis-state $configDir/genesis.ssz \ --validator-config $configDir/validator-config.yaml \ $devnet_flag \ @@ -54,11 +58,13 @@ node_binary="$scriptDir/lantern/build/lantern_cli \ --hash-sig-key-dir $configDir/hash-sig-keys \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" node_docker="$LANTERN_IMAGE --data-dir /data \ --genesis-config /config/config.yaml \ --validator-registry-path /config/validators.yaml \ + --validator-keys-path /config/annotated_validators.yaml \ --genesis-state /config/genesis.ssz \ --validator-config /config/validator-config.yaml \ $devnet_flag \ @@ -71,6 +77,7 @@ node_docker="$LANTERN_IMAGE --data-dir /data \ --hash-sig-key-dir /config/hash-sig-keys \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" # choose either binary or docker diff --git a/client-cmds/lighthouse-cmd.sh b/client-cmds/lighthouse-cmd.sh index eb45016..4d25b83 100644 --- a/client-cmds/lighthouse-cmd.sh +++ b/client-cmds/lighthouse-cmd.sh @@ -9,6 +9,15 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi +# In multi-subnet deployments, an aggregator must subscribe to every subnet's +# attestation topics so it can aggregate votes from all committees. The caller +# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the +# full subnet id set for the network. +aggregate_subnet_ids_flag="" +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then + aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" +fi + # Set attestation committee count flag if explicitly configured attestation_committee_flag="" if [ -n "$attestationCommitteeCount" ]; then @@ -36,6 +45,7 @@ node_binary="$lighthouse_bin lean_node \ --api-port $apiPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" node_docker="hopinheimer/lighthouse:latest lighthouse lean_node \ @@ -53,6 +63,7 @@ node_docker="hopinheimer/lighthouse:latest lighthouse lean_node \ --api-port $apiPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" node_setup="docker" diff --git a/client-cmds/nlean-cmd.sh b/client-cmds/nlean-cmd.sh index 670dbdd..9d28a5c 100755 --- a/client-cmds/nlean-cmd.sh +++ b/client-cmds/nlean-cmd.sh @@ -6,7 +6,7 @@ # NLEAN_REPO should point to this repository when lean-quickstart is outside this workspace. # Default assumes sibling checkouts: /nlean and /lean-quickstart. nlean_repo="${NLEAN_REPO:-$scriptDir/../nlean}" -nlean_docker_image="${NLEAN_DOCKER_IMAGE:-ghcr.io/nleaneth/nlean:devnet3}" +nlean_docker_image="${NLEAN_DOCKER_IMAGE:-ghcr.io/nleaneth/nlean:devnet4}" nlean_network_name="${NLEAN_NETWORK_NAME:-devnet0}" log_level="${NLEAN_LOG_LEVEL:-}" enable_metrics="${enableMetrics:-false}" @@ -35,6 +35,15 @@ if [[ "${isAggregator:-false}" == "true" ]]; then aggregator_flag="--is-aggregator" fi +# In multi-subnet deployments, an aggregator must subscribe to every subnet's +# attestation topics so it can aggregate votes from all committees. The caller +# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the +# full subnet id set for the network. +aggregate_subnet_ids_flag="" +if [[ "${isAggregator:-false}" == "true" ]] && [[ -n "${aggregateSubnetIds:-}" ]] && [[ "$aggregateSubnetIds" == *,* ]]; then + aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" +fi + # Set attestation committee count flag if explicitly configured attestation_committee_flag="" if [[ -n "${attestationCommitteeCount:-}" ]]; then @@ -71,6 +80,7 @@ node_binary="$binary_path \ $api_port_flag \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag \ $log_level_arg" @@ -112,6 +122,7 @@ node_docker="${nlean_docker_extra_env} ${nlean_docker_image} \ $api_port_flag \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag \ $log_level_arg" diff --git a/client-cmds/peam-cmd.sh b/client-cmds/peam-cmd.sh index 6c136a0..d1863b1 100644 --- a/client-cmds/peam-cmd.sh +++ b/client-cmds/peam-cmd.sh @@ -3,7 +3,7 @@ #-----------------------peam setup---------------------- binary_path="$scriptDir/../Peam/target/release/peam" -default_peam_docker_image="ghcr.io/malik672/peam:devnet3" +default_peam_docker_image="ghcr.io/malik672/peam:devnet4" peam_docker_image="${PEAM_DOCKER_IMAGE:-$default_peam_docker_image}" runtime_config_host="$dataDir/$item/peam.conf" runtime_config_container="/data/peam.conf" @@ -53,6 +53,18 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi +# In multi-subnet deployments, an aggregator must subscribe to every subnet's +# attestation topics so it can aggregate votes from all committees. The caller +# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the +# full subnet id set for the network. Note: peam already subscribes to all +# subnets in [0, committee_count) via allowed_topics above; this flag exists +# for contract parity with other clients and is a no-op unless the binary +# recognises it. +aggregate_subnet_ids_flag="" +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then + aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" +fi + checkpoint_sync_flag="" if [ -n "${checkpoint_sync_url:-}" ]; then checkpoint_sync_flag="--checkpoint-sync-url $checkpoint_sync_url" @@ -76,6 +88,7 @@ node_binary="$binary_path \ $validator_keys_flag \ $api_port_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" validator_keys_flag_container="" @@ -83,7 +96,7 @@ if [ -d "$hash_sig_keys_dir" ]; then validator_keys_flag_container="--validator-keys /config/hash-sig-keys" fi -node_docker="ghcr.io/malik672/peam:devnet3 \ +node_docker="ghcr.io/malik672/peam:devnet4 \ --run \ --config $runtime_config_container \ --data-dir /data \ @@ -91,10 +104,11 @@ node_docker="ghcr.io/malik672/peam:devnet3 \ $validator_keys_flag_container \ $api_port_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" if [ -n "${PEAM_DOCKER_IMAGE:-}" ]; then - node_docker="${peam_docker_image}${node_docker#"ghcr.io/malik672/peam:devnet3"}" + node_docker="${peam_docker_image}${node_docker#"ghcr.io/malik672/peam:devnet4"}" fi node_setup="docker" diff --git a/client-cmds/qlean-cmd.sh b/client-cmds/qlean-cmd.sh index f8b1f47..b228e6b 100644 --- a/client-cmds/qlean-cmd.sh +++ b/client-cmds/qlean-cmd.sh @@ -7,9 +7,9 @@ # Platform-specific qlean image ARCH=$(uname -m) if [ "$ARCH" = "x86_64" ]; then - QLEAN_IMAGE="qdrvm/qlean-mini:devnet-3-amd64" + QLEAN_IMAGE="qdrvm/qlean-mini:devnet-4-amd64" elif [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then - QLEAN_IMAGE="qdrvm/qlean-mini:devnet-3-arm64" + QLEAN_IMAGE="qdrvm/qlean-mini:devnet-4-arm64" else echo "Unsupported architecture: $ARCH" exit 1 @@ -21,6 +21,15 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi +# In multi-subnet deployments, an aggregator must subscribe to every subnet's +# attestation topics so it can aggregate votes from all committees. The caller +# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the +# full subnet id set for the network. +aggregate_subnet_ids_flag="" +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then + aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" +fi + # Set attestation committee count flag if explicitly configured attestation_committee_flag="" if [ -n "$attestationCommitteeCount" ]; then @@ -33,14 +42,8 @@ if [ -n "${checkpoint_sync_url:-}" ]; then checkpoint_sync_flag="--checkpoint-sync-url $checkpoint_sync_url" fi -node_binary="$scriptDir/qlean/build/src/executable/qlean \ - --modules-dir $scriptDir/qlean/build/src/modules \ - --genesis $configDir/config.yaml \ - --validator-registry-path $configDir/validators.yaml \ - --validator-keys-manifest $configDir/hash-sig-keys/validator-keys-manifest.yaml \ - --xmss-pk $hashSigPkPath \ - --xmss-sk $hashSigSkPath \ - --bootnodes $configDir/nodes.yaml \ +node_binary="$scriptDir/qlean/build/out/bin/qlean \ + --genesis-dir $configDir \ --data-dir $dataDir/$item \ --node-id $item --node-key $configDir/$privKeyPath \ --listen-addr /ip4/0.0.0.0/udp/$quicPort/quic-v1 \ @@ -50,17 +53,12 @@ node_binary="$scriptDir/qlean/build/src/executable/qlean \ --api-port $apiPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag \ - -ldebug \ - -ltrace" - + -ldebug" + node_docker="$QLEAN_IMAGE \ - --genesis /config/config.yaml \ - --validator-registry-path /config/validators.yaml \ - --validator-keys-manifest /config/hash-sig-keys/validator-keys-manifest.yaml \ - --xmss-pk /config/hash-sig-keys/validator_${hashSigKeyIndex}_pk.json \ - --xmss-sk /config/hash-sig-keys/validator_${hashSigKeyIndex}_sk.json \ - --bootnodes /config/nodes.yaml \ + --genesis-dir /config \ --data-dir /data \ --node-id $item --node-key /config/$privKeyPath \ --listen-addr /ip4/0.0.0.0/udp/$quicPort/quic-v1 \ @@ -70,9 +68,9 @@ node_docker="$QLEAN_IMAGE \ --api-port $apiPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag \ - -ldebug \ - -ltrace" + -ldebug" # choose either binary or docker node_setup="docker" diff --git a/client-cmds/ream-cmd.sh b/client-cmds/ream-cmd.sh index ef387bb..41315c4 100755 --- a/client-cmds/ream-cmd.sh +++ b/client-cmds/ream-cmd.sh @@ -10,6 +10,15 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi +# In multi-subnet deployments, an aggregator must subscribe to every subnet's +# attestation topics so it can aggregate votes from all committees. The caller +# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the +# full subnet id set for the network. +aggregate_subnet_ids_flag="" +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then + aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" +fi + # Set attestation committee count flag if explicitly configured attestation_committee_flag="" if [ -n "$attestationCommitteeCount" ]; then @@ -26,7 +35,7 @@ fi node_binary="$scriptDir/../ream/target/release/ream --data-dir $dataDir/$item \ lean_node \ --network $configDir/config.yaml \ - --validator-registry-path $configDir/validators.yaml \ + --validator-registry-path $configDir/annotated_validators.yaml \ --bootnodes $configDir/nodes.yaml \ --node-id $item --node-key $configDir/$privKeyPath \ --socket-port $quicPort \ @@ -37,12 +46,13 @@ node_binary="$scriptDir/../ream/target/release/ream --data-dir $dataDir/$item \ --http-port $apiPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" -node_docker="ghcr.io/reamlabs/ream:latest-devnet3 --data-dir /data \ +node_docker="ghcr.io/reamlabs/ream:latest-devnet4 --data-dir /data \ lean_node \ --network /config/config.yaml \ - --validator-registry-path /config/validators.yaml \ + --validator-registry-path /config/annotated_validators.yaml \ --bootnodes /config/nodes.yaml \ --node-id $item --node-key /config/$privKeyPath \ --socket-port $quicPort \ @@ -53,6 +63,7 @@ node_docker="ghcr.io/reamlabs/ream:latest-devnet3 --data-dir /data \ --http-port $apiPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" # choose either binary or docker diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index 9a31a25..61af2f2 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -7,7 +7,7 @@ metrics_flag="--metrics_enable" # Optional global zeam CLI flags before `node` (e.g. --console-log-level debug). -# Default empty: blockblaz/zeam:devnet3 and older binaries do not support top-level log flags. +# Default empty: blockblaz/zeam:devnet4 and older binaries do not support top-level log flags. # With a current zeam build: export ZEAM_GLOBAL_FLAGS='--console-log-level debug' zeam_global_flags="${ZEAM_GLOBAL_FLAGS:-}" @@ -17,6 +17,15 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi +# In multi-subnet deployments, an aggregator must subscribe to every subnet's +# attestation topics so it can aggregate votes from all committees. The caller +# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the +# full subnet id set for the network. +aggregate_subnet_ids_flag="" +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then + aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" +fi + # Set attestation committee count flag if explicitly configured attestation_committee_flag="" if [ -n "$attestationCommitteeCount" ]; then @@ -39,9 +48,10 @@ node_binary="$scriptDir/../zig-out/bin/zeam $zeam_global_flags node \ --metrics-port $metricsPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" -node_docker="--security-opt seccomp=unconfined blockblaz/zeam:devnet3 $zeam_global_flags node \ +node_docker="--security-opt seccomp=unconfined blockblaz/zeam:devnet4 $zeam_global_flags node \ --custom_genesis /config \ --validator_config $validatorConfig \ --data-dir /data \ @@ -51,6 +61,7 @@ node_docker="--security-opt seccomp=unconfined blockblaz/zeam:devnet3 $zeam_glob --metrics-port $metricsPort \ $attestation_committee_flag \ $aggregator_flag \ + $aggregate_subnet_ids_flag \ $checkpoint_sync_flag" # choose either binary or docker diff --git a/docs/adding-a-new-client.md b/docs/adding-a-new-client.md index 52203c8..e7485ef 100644 --- a/docs/adding-a-new-client.md +++ b/docs/adding-a-new-client.md @@ -208,14 +208,14 @@ Your client will find these files at `$configDir` (or `/config` in Docker): | File | Contents | |---|---| -| `config.yaml` | Chain config — genesis time, ACTIVE\_EPOCH, VALIDATOR\_COUNT, GENESIS\_VALIDATORS pubkeys | +| `config.yaml` | Chain config — genesis time, ATTESTATION\_COMMITTEE\_COUNT, VALIDATOR\_COUNT, GENESIS\_VALIDATORS (per-validator `attestation_pubkey` + `proposal_pubkey` for devnet4) | | `validators.yaml` | Validator index → node name assignments | -| `annotated_validators.yaml` | Validator index + pubkey\_hex + privkey\_file per node name (preferred over validators.yaml) | +| `annotated_validators.yaml` | Validator index + pubkey\_hex + privkey\_file per assignment (devnet4: two rows per index — attester + proposer `privkey_file`) | | `nodes.yaml` | ENR list for all nodes — use as static bootnode list | | `genesis.json` | Genesis state (JSON) | | `genesis.ssz` | Genesis state (SSZ) | -| `hash-sig-keys/validator_N_sk.ssz` | Post-quantum secret key for validator N | -| `hash-sig-keys/validator_N_pk.ssz` | Post-quantum public key for validator N | +| `hash-sig-keys/validator_N_attester_key_{sk,pk}.ssz` | Post-quantum attester secret/public key for validator N | +| `hash-sig-keys/validator_N_proposer_key_{sk,pk}.ssz` | Post-quantum proposer secret/public key for validator N | | `myclient_0.key` | P2P libp2p private key for this node | > Clients should derive their genesis state from `config.yaml` directly (using diff --git a/generate-ansible-inventory.sh b/generate-ansible-inventory.sh index e36044e..ad83b41 100755 --- a/generate-ansible-inventory.sh +++ b/generate-ansible-inventory.sh @@ -99,6 +99,28 @@ for node_name in "${nodes[@]}"; do fi done +# One inventory host per remote IP for prepare.yml — avoids N parallel SSH/apt sessions +# to the same machine when validator-config lists zeam_0..zeam_4 on one IP. +PREPARE_FILE="${OUTPUT_DIR}/hosts-prepare.yml" +cat > "$PREPARE_FILE" << 'EOF' +--- +# Deduplicated inventory for prepare.yml only (generated; do not edit manually). +all: + children: + prepare_hosts: + hosts: {} +EOF + +while IFS= read -r ip; do + [ -z "$ip" ] || [ "$ip" = "null" ] && continue + if [[ "$ip" == "127.0.0.1" ]] || [[ "$ip" == "localhost" ]]; then + continue + fi + inv_id="prep_${ip//./_}" + yq eval -i ".all.children.prepare_hosts.hosts.\"$inv_id\".ansible_host = \"$ip\"" "$PREPARE_FILE" +done < <(yq eval '.validators[].enrFields.ip' "$VALIDATOR_CONFIG" | sort -u) + echo "✅ Generated Ansible inventory at: $OUTPUT_FILE" +echo "✅ Generated prepare inventory (one host per IP) at: $PREPARE_FILE" echo " Processed ${#nodes[@]} node(s): ${nodes[*]}" diff --git a/generate-genesis.sh b/generate-genesis.sh index 422c3db..6bcf30a 100755 --- a/generate-genesis.sh +++ b/generate-genesis.sh @@ -162,8 +162,8 @@ if ! command -v docker &> /dev/null; then fi echo " ✅ docker found: $(which docker)" -# Hash-sig-cli Docker image -HASH_SIG_CLI_IMAGE="blockblaz/hash-sig-cli:devnet2" +# Hash-sig-cli Docker image (separate attester + proposer keys per validator when using dual-key manifest) +HASH_SIG_CLI_IMAGE="blockblaz/hash-sig-cli:latest" echo " ✅ Using hash-sig-cli Docker image: $HASH_SIG_CLI_IMAGE" echo "" @@ -211,8 +211,18 @@ if [ ! -f "$MANIFEST_FILE" ]; then KEYS_EXIST=false else for ((i=0; i/dev/null) +DUAL_KEY_MODE=false PUBKEY_FIELD="" -if echo "$FIRST_VALIDATOR_FIELDS" | grep -q "pubkey_hex"; then +if echo "$FIRST_VALIDATOR_FIELDS" | grep -q "attester_key_pubkey_hex" && \ + echo "$FIRST_VALIDATOR_FIELDS" | grep -q "proposer_key_pubkey_hex"; then + DUAL_KEY_MODE=true +elif echo "$FIRST_VALIDATOR_FIELDS" | grep -q "pubkey_hex"; then PUBKEY_FIELD="pubkey_hex" elif echo "$FIRST_VALIDATOR_FIELDS" | grep -q "public_key_file"; then PUBKEY_FIELD="public_key_file" elif echo "$FIRST_VALIDATOR_FIELDS" | grep -q "publicKey"; then PUBKEY_FIELD="publicKey" else - echo " ❌ Error: Could not determine pubkey field name in manifest" - echo " Expected 'pubkey_hex', 'public_key_file', or 'publicKey' field" - exit 1 -fi - -# Verify that manifest contains hex bytes (not file names) -FIRST_PUBKEY=$(yq eval ".validators[0].$PUBKEY_FIELD" "$MANIFEST_FILE" 2>/dev/null) -if [ -z "$FIRST_PUBKEY" ]; then - echo " ❌ Error: Could not read pubkey from manifest" + echo " ❌ Error: Could not determine manifest pubkey layout" + echo " Expected dual keys (attester_key_pubkey_hex + proposer_key_pubkey_hex) or legacy pubkey_hex / public_key_file / publicKey" exit 1 fi -# Check if it's hex format (starts with 0x) -if [[ ! "$FIRST_PUBKEY" =~ ^0x[0-9a-fA-F]+$ ]]; then - echo " ❌ Error: Manifest does not contain hex pubkeys" - echo " Found: $FIRST_PUBKEY" - echo " Expected format: 0x[hex bytes]" - echo " Make sure hash-sig-cli generates manifest with hex bytes" - exit 1 +if [ "$DUAL_KEY_MODE" = true ]; then + ATTEST_PUB=$(yq eval '.validators[0].attester_key_pubkey_hex' "$MANIFEST_FILE" 2>/dev/null) + PROP_PUB=$(yq eval '.validators[0].proposer_key_pubkey_hex' "$MANIFEST_FILE" 2>/dev/null) + if [ -z "$ATTEST_PUB" ] || [ -z "$PROP_PUB" ]; then + echo " ❌ Error: Could not read attester/proposer pubkeys from manifest" + exit 1 + fi + for pk in "$ATTEST_PUB" "$PROP_PUB"; do + if [[ ! "$pk" =~ ^0x[0-9a-fA-F]+$ ]]; then + echo " ❌ Error: Manifest does not contain hex pubkeys (dual-key mode)" + echo " Found: $pk" + exit 1 + fi + done + echo " ✅ Manifest verified - dual-key format (attester + proposer)" +else + FIRST_PUBKEY=$(yq eval ".validators[0].$PUBKEY_FIELD" "$MANIFEST_FILE" 2>/dev/null) + if [ -z "$FIRST_PUBKEY" ]; then + echo " ❌ Error: Could not read pubkey from manifest" + exit 1 + fi + if [[ ! "$FIRST_PUBKEY" =~ ^0x[0-9a-fA-F]+$ ]]; then + echo " ❌ Error: Manifest does not contain hex pubkeys" + echo " Found: $FIRST_PUBKEY" + echo " Expected format: 0x[hex bytes]" + exit 1 + fi + echo " ✅ Manifest verified - contains hex pubkeys (legacy)" + echo " Detected pubkey field: $PUBKEY_FIELD" fi -echo " ✅ Manifest verified - contains hex pubkeys" -echo " Detected pubkey field: $PUBKEY_FIELD" - echo "" # ======================================== @@ -390,11 +415,20 @@ done < <(yq eval '.validators[].name' "$VALIDATOR_CONFIG_FILE") echo " Total validator count: $TOTAL_VALIDATORS" +# Optional chain setting; default matches leanSpec ATTESTATION_COMMITTEE_COUNT (Uint64(1)) +ATTESTATION_COMMITTEE_COUNT=$(yq eval '.config.attestation_committee_count // 1' "$VALIDATOR_CONFIG_FILE" 2>/dev/null) +if [ -z "$ATTESTATION_COMMITTEE_COUNT" ] || [ "$ATTESTATION_COMMITTEE_COUNT" == "null" ]; then + ATTESTATION_COMMITTEE_COUNT=1 +fi + # Generate config.yaml from scratch cat > "$CONFIG_FILE" << EOF # Genesis Settings GENESIS_TIME: $GENESIS_TIME +# Chain Settings +ATTESTATION_COMMITTEE_COUNT: $ATTESTATION_COMMITTEE_COUNT + # Key Settings ACTIVE_EPOCH: $ACTIVE_EPOCH @@ -477,43 +511,59 @@ VALIDATOR_ENTRY_INDEX=0 # Create temporary file for genesis_validators YAML GENESIS_VALIDATORS_TMP=$(mktemp) -# Debug: show manifest file and pubkey field +# Debug: show manifest file and layout echo " Reading pubkeys from: $MANIFEST_FILE" -echo " Using pubkey field: $PUBKEY_FIELD" +if [ "$DUAL_KEY_MODE" = true ]; then + echo " Layout: dual-key (attestation_pubkey + proposal_pubkey in config.yaml)" +else + echo " Using pubkey field: $PUBKEY_FIELD" +fi # Iterate through validators in validator-config.yaml while IFS= read -r validator_name; do COUNT=$(yq eval ".validators[$VALIDATOR_ENTRY_INDEX].count" "$VALIDATOR_CONFIG_FILE") - - # Read hex pubkey directly from manifest (hash-sig-cli now generates hex) - PUBKEY_HEX=$(yq eval ".validators[$VALIDATOR_ENTRY_INDEX].$PUBKEY_FIELD" "$MANIFEST_FILE" 2>/dev/null) - - # Debug output - echo " Validator $VALIDATOR_ENTRY_INDEX ($validator_name): count=$COUNT, pubkey=${PUBKEY_HEX:0:20}..." - - if [ -z "$PUBKEY_HEX" ] || [ "$PUBKEY_HEX" == "null" ]; then - echo " ❌ Error: Could not read pubkey for validator $VALIDATOR_ENTRY_INDEX from manifest" - rm -f "$GENESIS_VALIDATORS_TMP" - exit 1 - fi - - # Verify it's hex format - if [[ ! "$PUBKEY_HEX" =~ ^0x[0-9a-fA-F]+$ ]]; then - echo " ❌ Error: Invalid pubkey format for validator $VALIDATOR_ENTRY_INDEX" - echo " Found: $PUBKEY_HEX" - echo " Expected format: 0x[hex bytes]" - rm -f "$GENESIS_VALIDATORS_TMP" - exit 1 - fi - - # For each validator index this entry represents + echo " Node $VALIDATOR_ENTRY_INDEX ($validator_name): count=$COUNT" + + # For each global validator index this node owns for ((idx=0; idx> "$GENESIS_VALIDATORS_TMP" + if [ "$DUAL_KEY_MODE" = true ]; then + AH=$(yq eval ".validators[$ACTUAL_INDEX].attester_key_pubkey_hex" "$MANIFEST_FILE" 2>/dev/null) + PH=$(yq eval ".validators[$ACTUAL_INDEX].proposer_key_pubkey_hex" "$MANIFEST_FILE" 2>/dev/null) + echo " index $ACTUAL_INDEX: attester=${AH:0:12}..., proposer=${PH:0:12}..." + if [ -z "$AH" ] || [ "$AH" == "null" ] || [ -z "$PH" ] || [ "$PH" == "null" ]; then + echo " ❌ Error: Could not read attester/proposer pubkeys for manifest index $ACTUAL_INDEX" + rm -f "$GENESIS_VALIDATORS_TMP" + exit 1 + fi + for pk in "$AH" "$PH"; do + if [[ ! "$pk" =~ ^0x[0-9a-fA-F]+$ ]]; then + echo " ❌ Error: Invalid pubkey format for validator index $ACTUAL_INDEX" + echo " Found: $pk" + rm -f "$GENESIS_VALIDATORS_TMP" + exit 1 + fi + done + echo " - attestation_pubkey: \"${AH#0x}\"" >> "$GENESIS_VALIDATORS_TMP" + echo " proposal_pubkey: \"${PH#0x}\"" >> "$GENESIS_VALIDATORS_TMP" + else + PK=$(yq eval ".validators[$ACTUAL_INDEX].$PUBKEY_FIELD" "$MANIFEST_FILE" 2>/dev/null) + echo " index $ACTUAL_INDEX: pubkey=${PK:0:20}..." + if [ -z "$PK" ] || [ "$PK" == "null" ]; then + echo " ❌ Error: Could not read pubkey for manifest index $ACTUAL_INDEX" + rm -f "$GENESIS_VALIDATORS_TMP" + exit 1 + fi + if [[ ! "$PK" =~ ^0x[0-9a-fA-F]+$ ]]; then + echo " ❌ Error: Invalid pubkey format for validator index $ACTUAL_INDEX" + echo " Found: $PK" + rm -f "$GENESIS_VALIDATORS_TMP" + exit 1 + fi + echo " - \"${PK#0x}\"" >> "$GENESIS_VALIDATORS_TMP" + fi done - + CUMULATIVE_INDEX=$((CUMULATIVE_INDEX + COUNT)) VALIDATOR_ENTRY_INDEX=$((VALIDATOR_ENTRY_INDEX + 1)) done < <(yq eval '.validators[].name' "$VALIDATOR_CONFIG_FILE") @@ -522,7 +572,7 @@ done < <(yq eval '.validators[].name' "$VALIDATOR_CONFIG_FILE") if [ -s "$GENESIS_VALIDATORS_TMP" ]; then # Append directly to config.yaml (simpler and more reliable than yq merge) echo "" >> "$CONFIG_FILE" - echo "# Genesis Validator Pubkeys" >> "$CONFIG_FILE" + echo "# List of Genesis Validators' Public Keys (attestation + proposal)" >> "$CONFIG_FILE" echo "GENESIS_VALIDATORS:" >> "$CONFIG_FILE" cat "$GENESIS_VALIDATORS_TMP" >> "$CONFIG_FILE" @@ -602,22 +652,40 @@ for idx in "${!ASSIGNMENT_NODE_NAMES[@]}"; do ENTRY_FOUND=true - PUBKEY_HEX_VALUE=$(yq eval ".validators[$raw_index].$PUBKEY_FIELD" "$MANIFEST_FILE" 2>/dev/null) - - if [ -z "$PUBKEY_HEX_VALUE" ] || [ "$PUBKEY_HEX_VALUE" == "null" ]; then - echo " ❌ Error: Missing pubkey for validator index $raw_index in manifest" - rm -f "$NODE_ASSIGNMENTS_TMP" - exit 1 - fi - - PUBKEY_HEX_NO_PREFIX="${PUBKEY_HEX_VALUE#0x}" - PRIVKEY_FILENAME="validator_${raw_index}_sk.ssz" - - cat << EOF >> "$NODE_ASSIGNMENTS_TMP" + if [ "$DUAL_KEY_MODE" = true ]; then + ATTEST_HEX_VALUE=$(yq eval ".validators[$raw_index].attester_key_pubkey_hex" "$MANIFEST_FILE" 2>/dev/null) + PROP_HEX_VALUE=$(yq eval ".validators[$raw_index].proposer_key_pubkey_hex" "$MANIFEST_FILE" 2>/dev/null) + if [ -z "$ATTEST_HEX_VALUE" ] || [ "$ATTEST_HEX_VALUE" == "null" ] || \ + [ -z "$PROP_HEX_VALUE" ] || [ "$PROP_HEX_VALUE" == "null" ]; then + echo " ❌ Error: Missing attester/proposer pubkey for validator index $raw_index in manifest" + rm -f "$NODE_ASSIGNMENTS_TMP" + exit 1 + fi + ATTEST_NO_PREFIX="${ATTEST_HEX_VALUE#0x}" + PROP_NO_PREFIX="${PROP_HEX_VALUE#0x}" + cat << EOF >> "$NODE_ASSIGNMENTS_TMP" + - index: $raw_index + pubkey_hex: $ATTEST_NO_PREFIX + privkey_file: validator_${raw_index}_attester_key_sk.ssz + - index: $raw_index + pubkey_hex: $PROP_NO_PREFIX + privkey_file: validator_${raw_index}_proposer_key_sk.ssz +EOF + else + PUBKEY_HEX_VALUE=$(yq eval ".validators[$raw_index].$PUBKEY_FIELD" "$MANIFEST_FILE" 2>/dev/null) + if [ -z "$PUBKEY_HEX_VALUE" ] || [ "$PUBKEY_HEX_VALUE" == "null" ]; then + echo " ❌ Error: Missing pubkey for validator index $raw_index in manifest" + rm -f "$NODE_ASSIGNMENTS_TMP" + exit 1 + fi + PUBKEY_HEX_NO_PREFIX="${PUBKEY_HEX_VALUE#0x}" + PRIVKEY_FILENAME="validator_${raw_index}_sk.ssz" + cat << EOF >> "$NODE_ASSIGNMENTS_TMP" - index: $raw_index pubkey_hex: $PUBKEY_HEX_NO_PREFIX privkey_file: $PRIVKEY_FILENAME EOF + fi done < <(yq eval "$INDEX_QUERY" "$VALIDATORS_OUTPUT_FILE" 2>/dev/null) if [ "$ENTRY_FOUND" = false ]; then @@ -710,8 +778,13 @@ done echo "" echo "🔐 Hash-Sig Validator Keys:" for i in $(seq 0 $((VALIDATOR_COUNT - 1))); do - echo " $GENESIS_DIR/hash-sig-keys/validator_${i}_pk.json" - echo " $GENESIS_DIR/hash-sig-keys/validator_${i}_sk.json" + if [ -f "$GENESIS_DIR/hash-sig-keys/validator_${i}_proposer_key_pk.json" ]; then + echo " $GENESIS_DIR/hash-sig-keys/validator_${i}_proposer_key_{pk,sk}.json" + echo " $GENESIS_DIR/hash-sig-keys/validator_${i}_attester_key_{pk,sk}.json" + else + echo " $GENESIS_DIR/hash-sig-keys/validator_${i}_pk.json" + echo " $GENESIS_DIR/hash-sig-keys/validator_${i}_sk.json" + fi done echo "" echo "🎯 Next steps:" diff --git a/generate-subnet-config.py b/generate-subnet-config.py index fe0283d..67a1d36 100644 --- a/generate-subnet-config.py +++ b/generate-subnet-config.py @@ -1,51 +1,42 @@ #!/usr/bin/env python3 """ -Generate an expanded validator-config.yaml from a template by distributing -each client across N subnets, one node per subnet per server. - -Subnet assignment rules ------------------------ - - Each server (IP) contributes exactly ONE node to each subnet. - - No two nodes on the same server share a subnet. - - Every subnet contains exactly the same number of clients. - - Every subnet contains at least one unique client (i.e. no two subnets - share a node identity). - -These rules are automatically satisfied by the expansion algorithm: the -template is expected to have one entry per client, each on its own server. -The script validates this assumption and errors out if it is violated. - -Port assignment ---------------- - For subnet i, all ports are incremented by i relative to the template entry: - quicPort += i - metricsPort += i - apiPort += i (or httpPort for Lantern) - - This keeps nodes on the same host from binding conflicting ports. - -Limits ------- - N must be between 1 and 5 (inclusive). - N=1 produces a single subnet (nodes renamed to {client}_0) with no port changes. +Generate an expanded validator-config.yaml from a template. + +Two modes (chosen automatically based on whether each client type appears once): + +1) **Replicate mode** (default) — each client type appears exactly once in the + template. Each row is cloned N times (subnets 0..N-1) with port offsets. + Works for both unique-IP deployments (Ansible) and shared-IP deployments + (local devnet where all clients run on 127.0.0.1). + + Port stride is ``i * len(template_validators)`` so no two clones ever + collide, even when all rows share the same IP. + +2) **Shared-host mode** — template has duplicate client types (multiple rows of + the same client, e.g. zeam_0..zeam_4 already present). No row cloning; each + row is one running node. Subnet membership is taken from each row's + ``subnet`` field, or inferred from a numeric name suffix ``client_K``. + + - **One client type per IP, many rows** (e.g. zeam_0..zeam_4 on 37.27.0.1): + infer ``subnet`` from the suffix K in ``name`` unless ``subnet`` is set. + - **Several client types on one IP** (zeam + ream + … on the same box, each + in a different subnet): every row for that IP **must** set explicit + ``subnet`` (name suffix is ambiguous when several clients use *_0). + +Limits: N (subnets / committee count) must be between 1 and 5. Usage ----- python3 generate-subnet-config.py - -Example -------- - python3 generate-subnet-config.py \\ - ansible-devnet/genesis/validator-config.yaml 2 \\ - ansible-devnet/genesis/validator-config-subnets-2.yaml """ from __future__ import annotations import copy +import re import secrets import sys -from collections import Counter +from collections import Counter, defaultdict import yaml @@ -57,78 +48,156 @@ def _client_name(node_name: str) -> str: return node_name.split("_")[0] -def _validate_template(validators: list[dict]) -> None: - """ - Enforce that the template satisfies the one-server-one-node requirement: - - No two entries share the same IP address. - - No two entries share the same client type (name prefix). - Either violation would break the subnet isolation guarantee. - """ - ips = [v["enrFields"]["ip"] for v in validators] +def _has_duplicate_clients(validators: list[dict]) -> bool: + """Return True when any client type appears more than once (shared-host template).""" clients = [_client_name(v["name"]) for v in validators] + return any(n > 1 for n in Counter(clients).values()) + - duplicate_ips = [ip for ip, n in Counter(ips).items() if n > 1] - if duplicate_ips: +def _subnet_from_name(name: str) -> int: + """Parse subnet index from trailing _ in node name.""" + m = re.match(r"^.+_(\d+)$", name) + if not m: + raise ValueError( + f"Node {name!r}: with shared-IP templates, use a numeric suffix " + f"(e.g. zeam_0) or set explicit 'subnet:'" + ) + return int(m.group(1)) + + +def _effective_subnet( + v: dict, + *, + ip: str, + clients_on_ip: set[str], +) -> int: + if "subnet" in v and v["subnet"] is not None and v["subnet"] != "": + return int(v["subnet"]) + if len(clients_on_ip) > 1: raise ValueError( - "Template validator-config.yaml has multiple entries sharing the " - f"same IP address: {duplicate_ips}. Each server must have exactly " - "one entry in the template. Use --subnets to add more nodes per server." + f"Node {v['name']!r} on {ip}: multiple client types share this IP; " + f"set an explicit integer 'subnet' on each row (name suffix alone is not enough)." ) + return _subnet_from_name(v["name"]) + + +def _validate_shared_host_template(validators: list[dict], n_subnets: int) -> None: + """Validate duplicate-IP (shared-host) layout and assign effective subnets.""" + by_ip: dict[str, list[dict]] = defaultdict(list) + for v in validators: + by_ip[v["enrFields"]["ip"]].append(v) + + seen_ip_subnet: set[tuple[str, int]] = set() + seen_ip_client_subnet: set[tuple[str, str, int]] = set() + + for ip, group in by_ip.items(): + clients_on_ip = {_client_name(v["name"]) for v in group} + for v in group: + client = _client_name(v["name"]) + try: + sn = _effective_subnet(v, ip=ip, clients_on_ip=clients_on_ip) + except ValueError: + raise + if sn < 0 or sn >= n_subnets: + raise ValueError( + f"Node {v['name']!r}: subnet {sn} out of range for --subnets {n_subnets} " + f"(valid: 0..{n_subnets - 1})" + ) + key = (ip, sn) + if key in seen_ip_subnet: + raise ValueError( + f"IP {ip}: two nodes use subnet {sn}; each subnet index must be " + f"unique per server (distinct ports / one node per subnet per host)." + ) + seen_ip_subnet.add(key) + + k2 = (ip, client, sn) + if k2 in seen_ip_client_subnet: + raise ValueError( + f"IP {ip}: duplicate entry for client {client!r} in subnet {sn}." + ) + seen_ip_client_subnet.add(k2) + + +def _expand_shared_host(template: dict, n_subnets: int) -> dict: + """Pass-through layout: one template row per node; set subnet + committee count.""" + validators = template["validators"] + _validate_shared_host_template(validators, n_subnets) + + by_ip: dict[str, list[dict]] = defaultdict(list) + for v in validators: + by_ip[v["enrFields"]["ip"]].append(v) + result = copy.deepcopy(template) + if "config" not in result: + result["config"] = {} + result["config"]["attestation_committee_count"] = n_subnets + + out_vals: list[dict] = [] + for v in validators: + entry = copy.deepcopy(v) + ip = entry["enrFields"]["ip"] + clients_on_ip = {_client_name(x["name"]) for x in by_ip[ip]} + sn = _effective_subnet(entry, ip=ip, clients_on_ip=clients_on_ip) + entry["subnet"] = sn + entry["isAggregator"] = False + out_vals.append(entry) + + result["validators"] = out_vals + return result + + +def _validate_replicate_template(validators: list[dict]) -> None: + """Each client type must appear exactly once (replicate mode clones them).""" + clients = [_client_name(v["name"]) for v in validators] duplicate_clients = [c for c, n in Counter(clients).items() if n > 1] if duplicate_clients: raise ValueError( "Template validator-config.yaml has multiple entries for the same " f"client type: {duplicate_clients}. Each client type must appear " - "exactly once in the template." + "exactly once in the template, or use a shared-host layout " + "(multiple rows per client type) instead." ) -def expand(template: dict, n_subnets: int) -> dict: +def expand_replicate(template: dict, n_subnets: int) -> dict: """ - Return a new config dict with every validator entry replicated across - n_subnets subnets. + Replicate mode: each template row becomes N nodes (subnets 0..N-1). - Output ordering: all subnet-0 nodes first, then all subnet-1 nodes, … - This makes the subnet grouping visually obvious in the generated file. + Port stride is ``i * len(validators)`` so no two clones collide, even when + all template rows share the same IP (e.g. a local devnet on 127.0.0.1). """ validators = template["validators"] - _validate_template(validators) + _validate_replicate_template(validators) result = copy.deepcopy(template) - # attestation_committee_count must equal the number of subnets so that - # each client correctly partitions itself into N separate committees. if "config" not in result: result["config"] = {} result["config"]["attestation_committee_count"] = n_subnets + stride = len(validators) expanded: list[dict] = [] for i in range(n_subnets): + offset = i * stride for validator in validators: client = _client_name(validator["name"]) - entry = copy.deepcopy(validator) + entry = copy.deepcopy(validator) - # Canonical name: {client}_{subnet_index} - entry["name"] = f"{client}_{i}" - entry["subnet"] = i # explicit membership for human readability + entry["name"] = f"{client}_{i}" + entry["subnet"] = i - # Every node beyond subnet 0 gets a fresh P2P identity key so - # nodes on the same server have different identities. if i > 0: entry["privkey"] = secrets.token_hex(32) - # Increment all network ports by the subnet index so nodes that - # share a host do not bind the same port. - entry["enrFields"]["quic"] = validator["enrFields"]["quic"] + i - entry["metricsPort"] = validator["metricsPort"] + i + entry["enrFields"]["quic"] = validator["enrFields"]["quic"] + offset + entry["metricsPort"] = validator["metricsPort"] + offset if "apiPort" in entry: - entry["apiPort"] = validator["apiPort"] + i + entry["apiPort"] = validator["apiPort"] + offset if "httpPort" in entry: - entry["httpPort"] = validator["httpPort"] + i + entry["httpPort"] = validator["httpPort"] + offset - # spin-node.sh re-assigns the aggregator before deploying. entry["isAggregator"] = False expanded.append(entry) @@ -137,13 +206,19 @@ def expand(template: dict, n_subnets: int) -> dict: return result +def expand(template: dict, n_subnets: int) -> dict: + if _has_duplicate_clients(template["validators"]): + return _expand_shared_host(template, n_subnets) + return expand_replicate(template, n_subnets) + + def main() -> None: if len(sys.argv) != 4: print(f"Usage: {sys.argv[0]} ") sys.exit(1) template_path = sys.argv[1] - output_path = sys.argv[3] + output_path = sys.argv[3] try: n_subnets = int(sys.argv[2]) @@ -172,14 +247,24 @@ def main() -> None: with open(output_path, "w") as fh: yaml.dump(expanded, fh, default_flow_style=False, sort_keys=False) - n_clients = len(template["validators"]) - n_nodes = len(expanded["validators"]) + n_in = len(template["validators"]) + n_out = len(expanded["validators"]) + mode = "shared-host" if _has_duplicate_clients(template["validators"]) else "replicate" print( f"Generated {output_path}:\n" - f" {n_clients} client(s) × {n_subnets} subnet(s) = {n_nodes} nodes\n" - f" config.attestation_committee_count = {n_subnets}\n" - f" Each server contributes exactly 1 node per subnet (no intra-server subnet sharing)" + f" mode = {mode}\n" + f" template rows = {n_in}, output nodes = {n_out}\n" + f" config.attestation_committee_count = {n_subnets}" ) + if mode == "replicate": + print( + f" (replicate) {n_in} client(s) × {n_subnets} subnet(s) = {n_out} nodes" + ) + else: + print( + " (shared-host) one output row per template row; subnets from " + "'subnet' field or numeric name suffix" + ) if __name__ == "__main__": diff --git a/local-devnet/genesis/validator-config.yaml b/local-devnet/genesis/validator-config.yaml index 1bb8af3..9547cd3 100644 --- a/local-devnet/genesis/validator-config.yaml +++ b/local-devnet/genesis/validator-config.yaml @@ -4,8 +4,8 @@ deployment_mode: local config: activeEpoch: 18 keyType: "hash-sig" - # Optional: Override client's hardcoded attestation_committee_count - # attestation_committee_count: 1 + # Optional; omit for default 1 (leanSpec src/lean_spec/subspecs/chain/config.py) + attestation_committee_count: 1 validators: - name: "zeam_0" privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" diff --git a/metrics/grafana/dashboards/client-dashboard.json b/metrics/grafana/dashboards/client-dashboard.json index 7c0a2ba..9c4aad3 100644 --- a/metrics/grafana/dashboards/client-dashboard.json +++ b/metrics/grafana/dashboards/client-dashboard.json @@ -153,7 +153,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max (lean_node_start_time_seconds{job=~\"$job\"} * 1000)", + "expr": "max by(network) (lean_node_start_time_seconds{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"} * 1000)", "instant": false, "legendFormat": "__auto", "range": true, @@ -196,7 +196,7 @@ "x": 4, "y": 1 }, - "id": 40, + "id": 103, "options": { "colorMode": "none", "graphMode": "none", @@ -221,14 +221,14 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(lean_attestation_committee_count{job=~\"$job\"})", + "expr": "sum by(network) (lean_validators_count{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], - "title": "Number of attestation committees", + "title": "Total number of validators", "type": "stat" }, { @@ -239,8 +239,7 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "super-light-green", - "mode": "fixed" + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -265,7 +264,7 @@ "x": 8, "y": 1 }, - "id": 89, + "id": 40, "options": { "colorMode": "none", "graphMode": "none", @@ -278,10 +277,7 @@ "values": false }, "showPercentChange": false, - "text": { - "valueSize": 30 - }, - "textMode": "name", + "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.2", @@ -293,14 +289,14 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max by(job) (lean_is_aggregator{job=~\"$job\"}) > 0", + "expr": "max by(network) (lean_attestation_committee_count{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "instant": true, - "legendFormat": "{{job}}", + "legendFormat": "__auto", "range": false, "refId": "A" } ], - "title": "Aggregator", + "title": "Number of attestation committees", "type": "stat" }, { @@ -345,8 +341,8 @@ }, "gridPos": { "h": 8, - "w": 6, - "x": 12, + "w": 4, + "x": 16, "y": 1 }, "id": 51, @@ -355,9 +351,7 @@ "displayMode": "table", "placement": "right", "showLegend": true, - "values": [ - "value" - ] + "values": [] }, "pieType": "donut", "reduceOptions": { @@ -370,7 +364,7 @@ "sort": "desc", "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -398,7 +392,7 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "sum by (job) (lean_validators_count{job=~\"$job\"})", + "expr": "sum by (network, job, instance) (lean_validators_count{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -410,9 +404,93 @@ "useBackend": false } ], - "title": "Validators per node", + "title": "Validators list", "type": "piechart" }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-green", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 89, + "options": { + "cellHeight": "sm", + "showHeader": false + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by(network,job,instance) (lean_is_aggregator{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}) > 0", + "instant": true, + "legendFormat": "{{job}}", + "range": false, + "refId": "A" + } + ], + "title": "Aggregators", + "transformations": [ + { + "id": "labelsToFields", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "job" + ] + } + } + } + ], + "type": "table" + }, { "datasource": { "type": "prometheus", @@ -477,8 +555,8 @@ }, "gridPos": { "h": 8, - "w": 6, - "x": 18, + "w": 4, + "x": 20, "y": 1 }, "id": 75, @@ -495,7 +573,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "last_over_time(lean_node_info{job=~\"$job\"}[1m])", + "expr": "last_over_time(lean_node_info{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[1m])", "instant": true, "legendFormat": "__auto", "range": false, @@ -523,12 +601,24 @@ "options": { "include": { "names": [ - "name", - "version", - "job" + "job", + "version" ] } } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "job": "Node", + "name": "Client", + "version": "Version" + } + } } ], "type": "table" @@ -591,7 +681,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(lean_latest_finalized_slot{job=~\"$job\"})", + "expr": "max by(network) (lean_latest_finalized_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "instant": true, "legendFormat": "__auto", "range": false, @@ -659,7 +749,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(lean_latest_justified_slot{job=~\"$job\"})", + "expr": "max by(network) (lean_latest_justified_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "instant": true, "legendFormat": "__auto", "range": false, @@ -727,7 +817,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(lean_head_slot{job=~\"$job\"})", + "expr": "max by(network) (lean_head_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "instant": true, "legendFormat": "__auto", "range": false, @@ -829,7 +919,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job)(lean_latest_finalized_slot{job=~\"$job\"})", + "expr": "sum by (network, job, instance)(lean_latest_finalized_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "interval": "", "legendFormat": "{{job}}", "range": true, @@ -931,7 +1021,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": " sum by (job) (lean_latest_justified_slot{job=~\"$job\"})", + "expr": " sum by (network, job, instance) (lean_latest_justified_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "legendFormat": "{{job}}", "range": true, "refId": "A" @@ -1032,7 +1122,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": " sum by (job) (lean_head_slot{job=~\"$job\"})", + "expr": " sum by (network, job, instance) (lean_head_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "legendFormat": "{{job}}", "range": true, "refId": "A" @@ -1133,7 +1223,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": " sum by (job) (lean_current_slot{job=~\"$job\"})", + "expr": " sum by (network, job, instance) (lean_current_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "legendFormat": "{{job}}", "range": true, "refId": "A" @@ -1207,7 +1297,7 @@ }, "gridPos": { "h": 8, - "w": 8, + "w": 12, "x": 0, "y": 23 }, @@ -1221,7 +1311,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -1234,7 +1324,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "changes(lean_node_start_time_seconds{job=~\"$job\"}[1m])", + "expr": "changes(lean_node_start_time_seconds{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[1m])", "instant": false, "interval": "", "legendFormat": "{{job}}", @@ -1254,7 +1344,6 @@ "fieldConfig": { "defaults": { "color": { - "fixedColor": "text", "mode": "palette-classic" }, "custom": { @@ -1291,7 +1380,6 @@ "mode": "off" } }, - "decimals": 0, "fieldMinMax": false, "mappings": [], "thresholds": { @@ -1320,11 +1408,11 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 8, + "w": 12, + "x": 12, "y": 23 }, - "id": 44, + "id": 90, "options": { "legend": { "calcs": [], @@ -1347,7 +1435,8 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": " sum by (job) (lean_connected_peers{job=~\"$job\"})", + "exemplar": false, + "expr": "sum by (network, job, instance) (lean_attestation_committee_subnet{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1359,7 +1448,7 @@ "useBackend": false } ], - "title": "Connected peers per node", + "title": "Attestation committee subnet", "type": "timeseries" }, { @@ -1371,6 +1460,7 @@ "fieldConfig": { "defaults": { "color": { + "fixedColor": "text", "mode": "palette-classic" }, "custom": { @@ -1407,6 +1497,7 @@ "mode": "off" } }, + "decimals": 0, "fieldMinMax": false, "mappings": [], "thresholds": { @@ -1435,11 +1526,11 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 16, - "y": 23 + "w": 12, + "x": 0, + "y": 31 }, - "id": 90, + "id": 44, "options": { "legend": { "calcs": [], @@ -1449,7 +1540,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -1462,35 +1553,21 @@ }, "disableTextWrap": false, "editorMode": "code", - "exemplar": false, - "expr": "sum by (job) (lean_attestation_committee_subnet{job=~\"$job\"})", + "expr": " sum by (network, job, instance) (lean_connected_peers{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "instant": true, + "instant": false, "interval": "", "legendFormat": "{{job}}", - "range": false, + "range": true, "refId": "B", "useBackend": false } ], - "title": "Attestation committee subnet", + "title": "Connected peers per node", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 57, - "panels": [], - "title": "Finalization/Justification Delay", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -1502,50 +1579,42 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", + "lineInterpolation": "stepAfter", + "lineWidth": 2, "fillOpacity": 0, - "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", "hideFrom": { "legend": false, "tooltip": false, "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } }, - "mappings": [], + "mappings": [ + { + "options": { + "0": { + "text": "idle" + }, + "1": { + "text": "syncing" + }, + "2": { + "text": "synced" + } + }, + "type": "value" + } + ], + "min": 0, + "max": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 + "value": null } ] } @@ -1553,26 +1622,20 @@ "overrides": [] }, "gridPos": { - "h": 10, - "w": 8, - "x": 0, - "y": 32 + "h": 8, + "w": 12, + "x": 12, + "y": 31 }, - "id": 30, + "id": 98, "options": { "legend": { - "calcs": [ - "min", - "max", - "mean", - "last" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "multi", "sort": "none" } @@ -1585,14 +1648,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time(lean_head_slot{job=~\"$job\"}[5m]) - avg_over_time(lean_latest_finalized_slot{job=~\"$job\"}[5m])", + "expr": "max by (network, job, instance) (\n (lean_node_sync_status{status=\"synced\", network=~\"$network\", job=~\"$job\", instance=~\"$instance\"} == 1) * 2\n or\n (lean_node_sync_status{status=\"syncing\", network=~\"$network\", job=~\"$job\", instance=~\"$instance\"} == 1) * 1\n or\n (lean_node_sync_status{status=\"idle\", network=~\"$network\", job=~\"$job\", instance=~\"$instance\"} == 1) * 0\n)", + "instant": false, "interval": "", "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Head - Finalized delay (slots)", + "title": "Node sync status", "type": "timeseries" }, { @@ -1600,6 +1664,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Total number of processed slots in state transition function", "fieldConfig": { "defaults": { "color": { @@ -1639,6 +1704,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -1657,21 +1723,16 @@ "overrides": [] }, "gridPos": { - "h": 10, - "w": 8, - "x": 8, - "y": 32 + "h": 8, + "w": 12, + "x": 0, + "y": 39 }, - "id": 29, + "id": 72, "options": { "legend": { - "calcs": [ - "min", - "max", - "mean", - "last" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", "showLegend": true }, @@ -1689,14 +1750,13 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time(lean_head_slot{job=~\"$job\"}[5m]) - avg_over_time(lean_latest_justified_slot{job=~\"$job\"}[5m])", - "interval": "", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_finalizations_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\", result=\"success\"}[1m]))[5m:]\n)", "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Head - Justified delay (slots)", + "title": "Finalizations - Success", "type": "timeseries" }, { @@ -1710,8 +1770,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", @@ -1743,6 +1801,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -1761,21 +1820,16 @@ "overrides": [] }, "gridPos": { - "h": 10, - "w": 8, - "x": 16, - "y": 32 + "h": 8, + "w": 12, + "x": 12, + "y": 39 }, - "id": 31, + "id": 73, "options": { "legend": { - "calcs": [ - "min", - "max", - "mean", - "last" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", "showLegend": true }, @@ -1793,14 +1847,13 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time(lean_latest_justified_slot{job=~\"$job\"}[5m]) - avg_over_time(lean_latest_finalized_slot{job=~\"$job\"}[5m])", - "interval": "", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_finalizations_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\", result=\"error\"}[1m]))[5m:]\n)", "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Justified - Finalized delay (slots)", + "title": "Finalizations - Errors", "type": "timeseries" }, { @@ -1809,11 +1862,1065 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 47 }, - "id": 53, + "id": 57, "panels": [], - "title": "Peers", + "title": "Finalization/Justification Delay", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 48 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(lean_head_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m]) - avg_over_time(lean_latest_finalized_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m])", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Head - Finalized delay (slots)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 48 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(lean_head_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m]) - avg_over_time(lean_latest_justified_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m])", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Head - Justified delay (slots)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 48 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(lean_latest_justified_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m]) - avg_over_time(lean_latest_finalized_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[5m])", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Justified - Finalized delay (slots)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 53, + "panels": [], + "title": "Peers", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": " sum by (network, job, instance) (lean_connected_peers{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Connected peers per node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": " sum by (network, job, instance, client) (lean_connected_peers{network=~\"$network\", job=~\"$job\", instance=~\"$instance\", client!=\"\"})", + "instant": false, + "interval": "", + "legendFormat": "{{job}} - {{client}}", + "range": true, + "refId": "A" + } + ], + "title": "Connected peers per node (detailed)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (network, job, instance) (\n increase(lean_peer_connection_events_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )", + "interval": "", + "legendFormat": "{{job}} {{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Peer connection events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 68 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (network, job, instance) (\n increase(lean_peer_disconnection_events_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )", + "interval": "", + "legendFormat": "{{job}} {{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Peer disconnection events", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 77 + }, + "id": 99, + "panels": [], + "title": "Gossip messages", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 78 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_gossip_block_size_bytes_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes size of a gossip block message", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 78 + }, + "id": 101, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_gossip_attestation_size_bytes_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes size of a gossip attestation message", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 78 + }, + "id": 102, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n rate(lean_gossip_aggregation_size_bytes_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Bytes size of a gossip aggregated attestation message", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 45, + "panels": [], + "title": "PQ Signatures", "type": "row" }, { @@ -1821,6 +2928,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Total number of individual attestation signatures", "fieldConfig": { "defaults": { "color": { @@ -1860,6 +2968,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -1878,12 +2987,12 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, + "h": 8, + "w": 8, "x": 0, - "y": 43 + "y": 87 }, - "id": 54, + "id": 60, "options": { "legend": { "calcs": [], @@ -1905,14 +3014,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": " sum by (job) (lean_connected_peers{job=~\"$job\"})", - "interval": "", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_attestation_signatures_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", + "instant": false, "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Connected peers per node", + "title": "Total number of attestation signatures", "type": "timeseries" }, { @@ -1920,6 +3030,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Total number of valid individual attestation signatures", "fieldConfig": { "defaults": { "color": { @@ -1959,6 +3070,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -1977,12 +3089,12 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 43 + "h": 8, + "w": 8, + "x": 8, + "y": 87 }, - "id": 73, + "id": 64, "options": { "legend": { "calcs": [], @@ -2005,15 +3117,14 @@ }, "editorMode": "code", "exemplar": false, - "expr": " sum by (job, client) (lean_connected_peers{job=~\"$job\", client!=\"\"})", + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_attestation_signatures_valid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", "instant": false, - "interval": "", - "legendFormat": "{{job}} - {{client}}", + "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Connected peers per node (detailed)", + "title": "Total number of valid attestation signatures", "type": "timeseries" }, { @@ -2021,6 +3132,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Total number of invalid individual attestation signatures", "fieldConfig": { "defaults": { "color": { @@ -2079,12 +3191,12 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 52 + "h": 8, + "w": 8, + "x": 16, + "y": 87 }, - "id": 55, + "id": 65, "options": { "legend": { "calcs": [], @@ -2106,14 +3218,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job) (\n increase(lean_peer_connection_events_total{job=~\"$job\"}[$__rate_interval])\n )", - "interval": "", - "legendFormat": "{{job}} {{source}}", + "exemplar": false, + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_attestation_signatures_invalid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", + "instant": false, + "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Peer connection events", + "title": "Total number of invalid attestation signatures", "type": "timeseries" }, { @@ -2121,6 +3234,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -2174,17 +3288,18 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 9, + "h": 8, "w": 12, - "x": 12, - "y": 52 + "x": 0, + "y": 95 }, - "id": 56, + "id": 46, "options": { "legend": { "calcs": [], @@ -2206,35 +3321,126 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job) (\n increase(lean_peer_disconnection_events_total{job=~\"$job\"}[$__rate_interval])\n )", - "interval": "", - "legendFormat": "{{job}} {{source}}", + "exemplar": false, + "expr": "histogram_quantile(0.99, rate(lean_pq_sig_attestation_signing_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Peer disconnection events", + "title": "Time taken to sign an attestation", "type": "timeseries" }, { - "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 61 + "h": 8, + "w": 12, + "x": 12, + "y": 95 }, - "id": 45, - "panels": [], - "title": "PQ Signatures", - "type": "row" + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, rate(lean_pq_sig_attestation_verification_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Time taken to verify an attestation signature", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "Total number of individual attestation signatures", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -2294,11 +3500,11 @@ }, "gridPos": { "h": 8, - "w": 8, + "w": 12, "x": 0, - "y": 62 + "y": 103 }, - "id": 60, + "id": 79, "options": { "legend": { "calcs": [], @@ -2321,14 +3527,14 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(\n sum by (job) (\n increase(lean_pq_sig_attestation_signatures_total{job=~\"$job\"}[$__rate_interval])\n )[5m:]\n)", + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_aggregated_signatures_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", "instant": false, "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Total number of attestation signatures", + "title": "Total number of aggregated signatures", "type": "timeseries" }, { @@ -2336,7 +3542,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Total number of valid individual attestation signatures", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -2396,11 +3602,11 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 8, - "y": 62 + "w": 12, + "x": 12, + "y": 103 }, - "id": 64, + "id": 61, "options": { "legend": { "calcs": [], @@ -2423,14 +3629,14 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(\n sum by (job) (\n increase(lean_pq_sig_attestation_signatures_valid_total{job=~\"$job\"}[$__rate_interval])\n )[5m:]\n)", + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_attestations_in_aggregated_signatures_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", "instant": false, "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Total number of valid attestation signatures", + "title": "Total number of attestations included into aggregated signatures", "type": "timeseries" }, { @@ -2438,7 +3644,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Total number of invalid individual attestation signatures", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -2498,11 +3704,11 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 16, - "y": 62 + "w": 12, + "x": 0, + "y": 111 }, - "id": 65, + "id": 77, "options": { "legend": { "calcs": [], @@ -2525,14 +3731,14 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(\n sum by (job) (\n increase(lean_pq_sig_attestation_signatures_invalid_total{job=~\"$job\"}[$__rate_interval])\n )[5m:]\n)", + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_aggregated_signatures_valid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", "instant": false, "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Total number of invalid attestation signatures", + "title": "Total number of valid aggregated signatures", "type": "timeseries" }, { @@ -2594,18 +3800,17 @@ "value": 80 } ] - }, - "unit": "s" + } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 70 + "x": 12, + "y": 111 }, - "id": 46, + "id": 78, "options": { "legend": { "calcs": [], @@ -2628,14 +3833,14 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.99, (rate(lean_pq_sig_attestation_signing_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])))", + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_pq_sig_aggregated_signatures_invalid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", "instant": false, "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Time taken to sign an attestation", + "title": "Total number of invalid aggregated signatures", "type": "timeseries" }, { @@ -2705,10 +3910,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 70 + "x": 0, + "y": 119 }, - "id": 47, + "id": 62, "options": { "legend": { "calcs": [], @@ -2731,14 +3936,14 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.99, (rate(lean_pq_sig_attestation_verification_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, rate(lean_pq_sig_aggregated_signatures_building_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Time taken to verify an attestation signature", + "title": "Time taken to build an aggregated signature", "type": "timeseries" }, { @@ -2800,17 +4005,18 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 78 + "x": 12, + "y": 119 }, - "id": 79, + "id": 63, "options": { "legend": { "calcs": [], @@ -2833,16 +4039,29 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(\n sum by (job) (\n increase(lean_pq_sig_aggregated_signatures_total{job=~\"$job\"}[$__rate_interval])\n )[5m:]\n)", + "expr": "histogram_quantile(0.99, rate(lean_pq_sig_aggregated_signatures_verification_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Total number of aggregated signatures", + "title": "Time taken to verify an aggregated signature", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 127 + }, + "id": 91, + "panels": [], + "title": "Block production", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -2888,7 +4107,6 @@ "mode": "off" } }, - "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -2909,10 +4127,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 78 + "x": 0, + "y": 128 }, - "id": 61, + "id": 19, "options": { "legend": { "calcs": [], @@ -2934,15 +4152,14 @@ "uid": "${datasource}" }, "editorMode": "code", - "exemplar": false, - "expr": "avg_over_time(\n sum by (job) (\n increase(lean_pq_sig_attestations_in_aggregated_signatures_total{job=~\"$job\"}[$__rate_interval])\n )[5m:]\n)", - "instant": false, + "expr": "histogram_quantile(0.99, \n rate(lean_block_aggregated_payloads_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Total number of attestations included into aggregated signatures", + "title": "Number of aggregated payloads", "type": "timeseries" }, { @@ -2990,7 +4207,6 @@ "mode": "off" } }, - "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -3004,17 +4220,18 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 86 + "x": 12, + "y": 128 }, - "id": 77, + "id": 93, "options": { "legend": { "calcs": [], @@ -3036,15 +4253,14 @@ "uid": "${datasource}" }, "editorMode": "code", - "exemplar": false, - "expr": "avg_over_time(\n sum by (job) (\n increase(lean_pq_sig_aggregated_signatures_valid_total{job=~\"$job\"}[$__rate_interval])\n )[5m:]\n)", - "instant": false, + "expr": "histogram_quantile(0.99, \n rate(lean_block_building_payload_aggregation_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Total number of valid aggregated signatures", + "title": "Time taken to build aggregated payloads", "type": "timeseries" }, { @@ -3052,7 +4268,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -3112,11 +4327,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 86 + "w": 8, + "x": 0, + "y": 136 }, - "id": 78, + "id": 80, "options": { "legend": { "calcs": [], @@ -3139,14 +4354,15 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(\n sum by (job) (\n increase(lean_pq_sig_aggregated_signatures_invalid_total{job=~\"$job\"}[$__rate_interval])\n )[5m:]\n)", + "expr": " avg_over_time (\n sum by (network, job, instance) (\n (lean_block_building_success_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})\n )[2m:]\n)", "instant": false, + "interval": "", "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Total number of invalid aggregated signatures", + "title": "Successful block builds", "type": "timeseries" }, { @@ -3154,7 +4370,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -3208,18 +4423,17 @@ "value": 80 } ] - }, - "unit": "s" + } }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 94 + "w": 8, + "x": 8, + "y": 136 }, - "id": 62, + "id": 97, "options": { "legend": { "calcs": [], @@ -3242,14 +4456,15 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.99, (rate(lean_pq_sig_attestation_signatures_building_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])))", + "expr": " avg_over_time (\n sum by (network, job, instance) (\n (lean_block_building_failures_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})\n )[2m:]\n)", "instant": false, + "interval": "", "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Time taken to build an aggregated signature", + "title": "Failed block builds", "type": "timeseries" }, { @@ -3297,7 +4512,6 @@ "mode": "off" } }, - "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -3318,11 +4532,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 94 + "w": 8, + "x": 16, + "y": 136 }, - "id": 63, + "id": 94, "options": { "legend": { "calcs": [], @@ -3344,15 +4558,14 @@ "uid": "${datasource}" }, "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, (rate(lean_pq_sig_aggregated_signatures_verification_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])))", - "instant": false, + "expr": "histogram_quantile(0.99, \n rate(lean_block_building_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", + "interval": "", "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Time taken to verify an aggregated signature", + "title": "Time taken to build a block", "type": "timeseries" }, { @@ -3361,7 +4574,7 @@ "h": 1, "w": 24, "x": 0, - "y": 102 + "y": 144 }, "id": 17, "panels": [], @@ -3429,40 +4642,15 @@ }, "unit": "s" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "zeam" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": true, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 103 + "y": 145 }, - "id": 19, + "id": 92, "options": { "legend": { "calcs": [], @@ -3484,7 +4672,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, \n rate(lean_fork_choice_block_processing_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n)", + "expr": "histogram_quantile(0.99, \n rate(lean_fork_choice_block_processing_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", "interval": "", "legendFormat": "{{job}}", "range": true, @@ -3561,7 +4749,7 @@ "h": 8, "w": 12, "x": 12, - "y": 103 + "y": 145 }, "id": 85, "options": { @@ -3585,7 +4773,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, \n rate(lean_committee_signatures_aggregation_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n)", + "expr": "histogram_quantile(0.99, \n rate(lean_committee_signatures_aggregation_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", "interval": "", "legendFormat": "{{job}}", "range": true, @@ -3661,7 +4849,7 @@ "h": 8, "w": 12, "x": 0, - "y": 111 + "y": 153 }, "id": 68, "options": { @@ -3685,7 +4873,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job) (\n increase(lean_fork_choice_reorgs_total{job=~\"$job\"}[$__rate_interval])\n)", + "expr": "sum by (network, job, instance) (\n increase(lean_fork_choice_reorgs_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", "interval": "", "legendFormat": "{{job}} {{source}}", "range": true, @@ -3761,7 +4949,7 @@ "h": 8, "w": 12, "x": 12, - "y": 111 + "y": 153 }, "id": 69, "options": { @@ -3785,7 +4973,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job) (\n increase(lean_fork_choice_reorg_depth_bucket{job=~\"$job\"}[$__rate_interval])\n)", + "expr": "histogram_quantile(0.99, rate(lean_fork_choice_reorg_depth_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "interval": "", "legendFormat": "{{job}} {{source}}", "range": true, @@ -3861,9 +5049,9 @@ "h": 8, "w": 8, "x": 0, - "y": 119 + "y": 161 }, - "id": 80, + "id": 96, "options": { "legend": { "calcs": [], @@ -3886,7 +5074,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": " avg_over_time (\n sum by (job) (\n (lean_gossip_signatures{job=~\"$job\"})\n )[2m:]\n)", + "expr": " avg_over_time (\n sum by (network, job, instance) (\n (lean_gossip_signatures{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})\n )[2m:]\n)", "instant": false, "interval": "", "legendFormat": "{{job}}", @@ -3963,7 +5151,7 @@ "h": 8, "w": 8, "x": 8, - "y": 119 + "y": 161 }, "id": 81, "options": { @@ -3988,7 +5176,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(\n sum by (job) (\n (lean_latest_new_aggregated_payloads{job=~\"$job\"})\n )[2m:]\n)", + "expr": "avg_over_time(\n sum by (network, job, instance) (\n (lean_latest_new_aggregated_payloads{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})\n )[2m:]\n)", "instant": false, "interval": "", "legendFormat": "{{job}}", @@ -4065,7 +5253,7 @@ "h": 8, "w": 8, "x": 16, - "y": 119 + "y": 161 }, "id": 82, "options": { @@ -4090,7 +5278,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(\n sum by (job) (\n (lean_latest_known_aggregated_payloads{job=~\"$job\"})\n )[1m:]\n)", + "expr": "avg_over_time(\n sum by (network, job, instance) (\n (lean_latest_known_aggregated_payloads{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})\n )[1m:]\n)", "instant": false, "interval": "", "legendFormat": "{{job}}", @@ -4107,7 +5295,7 @@ "h": 1, "w": 24, "x": 0, - "y": 127 + "y": 169 }, "id": 8, "panels": [], @@ -4180,7 +5368,7 @@ "h": 8, "w": 8, "x": 0, - "y": 128 + "y": 170 }, "id": 9, "options": { @@ -4204,7 +5392,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time(\n sum by (job) (\n increase(lean_attestations_valid_total{job=~\"$job\"}[$__rate_interval])\n )[5m:]\n)", + "expr": "avg_over_time(\n sum by (network, job, instance) (\n increase(lean_attestations_valid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n )[5m:]\n)", "interval": "", "legendFormat": "{{job}} {{source}}", "range": true, @@ -4279,7 +5467,7 @@ "h": 8, "w": 8, "x": 8, - "y": 128 + "y": 170 }, "id": 10, "options": { @@ -4303,7 +5491,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time(\n sum by (job) (increase(lean_attestations_invalid_total{job=~\"$job\"}[$__rate_interval]))[5m:]\n)", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_attestations_invalid_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))[5m:]\n)", "legendFormat": "{{job}} {{source}}", "range": true, "refId": "A" @@ -4378,7 +5566,7 @@ "h": 8, "w": 8, "x": 16, - "y": 128 + "y": 170 }, "id": 11, "options": { @@ -4402,7 +5590,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(lean_attestation_validation_time_seconds_bucket{job=~\"$job\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.99, rate(lean_attestation_validation_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{job}}", "range": true, "refId": "A" @@ -4477,7 +5665,7 @@ "h": 8, "w": 8, "x": 0, - "y": 136 + "y": 178 }, "id": 67, "options": { @@ -4501,7 +5689,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": " sum by (job) (lean_safe_target_slot{job=~\"$job\"})", + "expr": " sum by (network, job, instance) (lean_safe_target_slot{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"})", "legendFormat": "{{job}}", "range": true, "refId": "A" @@ -4577,7 +5765,7 @@ "h": 8, "w": 8, "x": 8, - "y": 136 + "y": 178 }, "id": 27, "options": { @@ -4602,7 +5790,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg_over_time(\n sum by (job) (increase(lean_state_transition_attestations_processed_total{job=~\"$job\"}[$__rate_interval]))[5m:]\n)", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_state_transition_attestations_processed_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))[5m:]\n)", "instant": false, "legendFormat": "{{job}}", "range": true, @@ -4679,7 +5867,7 @@ "h": 8, "w": 8, "x": 16, - "y": 136 + "y": 178 }, "id": 28, "options": { @@ -4703,7 +5891,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99,\n rate(lean_state_transition_attestations_processing_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n)", + "expr": "histogram_quantile(0.99,\n rate(lean_state_transition_attestations_processing_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", "legendFormat": "{{job}}", "range": true, "refId": "A" @@ -4770,7 +5958,8 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, @@ -4778,7 +5967,7 @@ "h": 8, "w": 12, "x": 0, - "y": 144 + "y": 186 }, "id": 83, "options": { @@ -4802,14 +5991,14 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(lean_attestations_production_time_seconds_bucket{job=~\"$job\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.99, rate(lean_attestations_production_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "interval": "", "legendFormat": "{{job}} {{source}}", "range": true, "refId": "A" } ], - "title": "Time taken to produce attestations", + "title": "Time taken to produce attestation", "type": "timeseries" }, { @@ -4878,7 +6067,7 @@ "h": 8, "w": 12, "x": 12, - "y": 144 + "y": 186 }, "id": 84, "options": { @@ -4902,7 +6091,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(lean_committee_signatures_aggregation_time_seconds_bucket{job=~\"$job\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.99, rate(lean_committee_signatures_aggregation_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{job}}", "range": true, "refId": "A" @@ -4917,7 +6106,7 @@ "h": 1, "w": 24, "x": 0, - "y": 152 + "y": 194 }, "id": 21, "panels": [], @@ -4991,7 +6180,7 @@ "h": 8, "w": 12, "x": 0, - "y": 153 + "y": 195 }, "id": 23, "options": { @@ -5015,7 +6204,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, \n rate(lean_state_transition_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n)", + "expr": "histogram_quantile(0.99, \n rate(lean_state_transition_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", "interval": "", "legendFormat": "{{job}}", "range": true, @@ -5092,7 +6281,7 @@ "h": 8, "w": 12, "x": 12, - "y": 153 + "y": 195 }, "id": 24, "options": { @@ -5116,7 +6305,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, \n rate(lean_state_transition_block_processing_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n)", + "expr": "histogram_quantile(0.99, \n rate(lean_state_transition_block_processing_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", "interval": "", "legendFormat": "{{job}}", "range": true, @@ -5193,7 +6382,7 @@ "h": 8, "w": 12, "x": 0, - "y": 161 + "y": 203 }, "id": 25, "options": { @@ -5217,7 +6406,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time(\n sum by (job) (increase(lean_state_transition_slots_processed_total{job=~\"$job\"}[$__rate_interval]))[5m:]\n)", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_state_transition_slots_processed_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))[5m:]\n)", "legendFormat": "{{job}}", "range": true, "refId": "A" @@ -5293,7 +6482,7 @@ "h": 8, "w": 12, "x": 12, - "y": 161 + "y": 203 }, "id": 26, "options": { @@ -5317,7 +6506,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99,\n rate(lean_state_transition_slots_processing_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])\n)", + "expr": "histogram_quantile(0.99,\n rate(lean_state_transition_slots_processing_time_seconds_bucket{network=~\"$network\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n)", "legendFormat": "{{job}}", "range": true, "refId": "A" @@ -5393,7 +6582,7 @@ "h": 8, "w": 12, "x": 0, - "y": 169 + "y": 211 }, "id": 70, "options": { @@ -5417,34 +6606,111 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time(\n sum by (job) (increase(lean_finalizations_total{job=~\"$job\"}[$__rate_interval]))[5m:]\n)", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_finalizations_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\", result=\"success\"}[1m]))[5m:]\n)", "legendFormat": "{{job}}", "range": true, "refId": "A" } ], - "title": "Finalizations total", + "title": "Finalizations - Success", "type": "timeseries" }, { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 169 + "y": 211 }, - "id": 52, + "id": 71, "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "content": "", - "mode": "markdown" + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } }, "pluginVersion": "12.3.2", - "type": "text" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(\n sum by (network, job, instance) (increase(lean_finalizations_total{network=~\"$network\", job=~\"$job\", instance=~\"$instance\", result=\"error\"}[1m]))[5m:]\n)", + "legendFormat": "{{job}}", + "range": true, + "refId": "A" + } + ], + "title": "Finalizations - Errors", + "type": "timeseries" } ], "preload": false, @@ -5458,7 +6724,23 @@ { "allowCustomValue": false, "current": {}, - "definition": "label_values(job)", + "definition": "label_values(network)", + "label": "Network", + "name": "network", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(network)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allowCustomValue": false, + "current": {}, + "definition": "label_values(lean_node_info{network=~\"$network\"}, job)", "includeAll": true, "label": "Job", "multi": true, @@ -5466,13 +6748,31 @@ "options": [], "query": { "qryType": 1, - "query": "label_values(job)", + "query": "label_values(lean_node_info{network=~\"$network\"}, job)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": ".*ethlambda.*|.*gean.*|.*grandine.*|.*lantern.*|.*lighthouse.*|.*qlean.*|.*ream.*|.*zeam.*", "type": "query" }, + { + "allowCustomValue": false, + "current": {}, + "definition": "label_values(lean_node_info{network=~\"$network\", job=~\"$job\"}, instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(lean_node_info{network=~\"$network\", job=~\"$job\"}, instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, { "allowCustomValue": false, "current": { @@ -5490,14 +6790,14 @@ ] }, "time": { - "from": "now-3h", + "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "utc", "title": "Lean Ethereum Clients Dashboard", "uid": "lean-ethereum-clients-dashboard", - "version": 16, + "version": 25, "weekStart": "", "id": null -} \ No newline at end of file +} diff --git a/parse-env.sh b/parse-env.sh index 51e56e1..50fc46c 100755 --- a/parse-env.sh +++ b/parse-env.sh @@ -173,8 +173,6 @@ fi; # freshStart logic removed - now handled by --generateGenesis flag -networkName="${networkName:-devnet-3}" - echo "configDir = $configDir" echo "dataDir = $dataDir" echo "spin_nodes(s) = ${spin_nodes[@]}" diff --git a/parse-vc.sh b/parse-vc.sh index 4fae8d2..6295221 100644 --- a/parse-vc.sh +++ b/parse-vc.sh @@ -69,6 +69,17 @@ if [ -z "$isAggregator" ] || [ "$isAggregator" == "null" ]; then isAggregator="false" fi +# Compute the full set of unique subnet ids as a CSV (e.g. "0,1"). Aggregators +# in multi-subnet deployments must subscribe to every subnet's attestation +# topic to aggregate votes across committees. Client-cmd scripts pass this +# along as --aggregate-subnet-ids when the node is an aggregator and the +# network has more than one subnet. +aggregateSubnetIds=$(yq eval '[.validators[].subnet // 0] | unique | sort | join(",")' "$validator_config_file") +if [ -z "$aggregateSubnetIds" ] || [ "$aggregateSubnetIds" == "null" ]; then + aggregateSubnetIds="" +fi +export aggregateSubnetIds + # Extract attestation_committee_count from config section (optional - only if explicitly set) attestationCommitteeCount=$(yq eval ".config.attestation_committee_count" "$validator_config_file") if [ -z "$attestationCommitteeCount" ] || [ "$attestationCommitteeCount" == "null" ]; then @@ -94,22 +105,39 @@ hashSigKeyIndex=$(yq eval ".validators | to_entries | .[] | select(.value.name = # Load hash-sig keys if configured if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$hashSigKeyIndex" ]; then - # Set hash-sig key paths - hashSigPkPath="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_pk.json" - hashSigSkPath="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_sk.json" - - # Validate that hash-sig keys exist - if [ ! -f "$hashSigPkPath" ]; then - echo "Warning: Hash-sig public key not found at $hashSigPkPath" - echo "Run genesis generator to create hash-sig keys: ./generate-genesis.sh $configDir" - fi - - if [ ! -f "$hashSigSkPath" ]; then - echo "Warning: Hash-sig secret key not found at $hashSigSkPath" - echo "Run genesis generator to create hash-sig keys: ./generate-genesis.sh $configDir" + # devnet4+: separate proposer + attester keys (hash-sig-cli); legacy: single pk/sk per index + _proposer_pk="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_proposer_key_pk.json" + _proposer_sk="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_proposer_key_sk.json" + _attester_pk="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_attester_key_pk.json" + _attester_sk="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_attester_key_sk.json" + _legacy_pk="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_pk.json" + _legacy_sk="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_sk.json" + + if [ -f "$_proposer_pk" ] && [ -f "$_attester_pk" ]; then + hashSigPkPath="$_proposer_pk" + hashSigSkPath="$_proposer_sk" + export HASH_SIG_PROPOSER_PK_PATH="$_proposer_pk" + export HASH_SIG_PROPOSER_SK_PATH="$_proposer_sk" + export HASH_SIG_ATTESTER_PK_PATH="$_attester_pk" + export HASH_SIG_ATTESTER_SK_PATH="$_attester_sk" + if [ ! -f "$hashSigSkPath" ] || [ ! -f "$_attester_sk" ]; then + echo "Warning: Hash-sig secret key(s) missing for dual-key layout (validator_${hashSigKeyIndex})" + echo "Run genesis generator: ./generate-genesis.sh $configDir" + fi + else + hashSigPkPath="$_legacy_pk" + hashSigSkPath="$_legacy_sk" + if [ ! -f "$hashSigPkPath" ]; then + echo "Warning: Hash-sig public key not found at $hashSigPkPath" + echo "Run genesis generator to create hash-sig keys: ./generate-genesis.sh $configDir" + fi + if [ ! -f "$hashSigSkPath" ]; then + echo "Warning: Hash-sig secret key not found at $hashSigSkPath" + echo "Run genesis generator to create hash-sig keys: ./generate-genesis.sh $configDir" + fi fi - - # Export hash-sig key paths for client use + + # Export hash-sig key paths for client use (HASH_SIG_PK_PATH = proposer when dual-key) export HASH_SIG_PK_PATH="$hashSigPkPath" export HASH_SIG_SK_PATH="$hashSigSkPath" export HASH_SIG_KEY_INDEX="$hashSigKeyIndex" @@ -124,6 +152,10 @@ if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$ha echo "Hash-Sig Key Index: $hashSigKeyIndex" echo "Hash-Sig Public Key: $hashSigPkPath" echo "Hash-Sig Secret Key: $hashSigSkPath" + if [ -n "${HASH_SIG_ATTESTER_PK_PATH:-}" ]; then + echo "Hash-Sig Attester PK: $HASH_SIG_ATTESTER_PK_PATH" + echo "Hash-Sig Attester SK: $HASH_SIG_ATTESTER_SK_PATH" + fi echo "Is Aggregator: $isAggregator" if [ -n "$attestationCommitteeCount" ]; then echo "Attestation Committee Count: $attestationCommitteeCount" diff --git a/run-ansible.sh b/run-ansible.sh index 4035df9..337091d 100755 --- a/run-ansible.sh +++ b/run-ansible.sh @@ -54,39 +54,60 @@ echo "SSH user for remote connections: $sshUser" # Generate ansible inventory from validator-config.yaml ANSIBLE_DIR="$scriptDir/ansible" INVENTORY_FILE="$ANSIBLE_DIR/inventory/hosts.yml" +PREPARE_INVENTORY="$ANSIBLE_DIR/inventory/hosts-prepare.yml" -# Generate inventory if it doesn't exist or if validator config is newer -if [ ! -f "$INVENTORY_FILE" ] || [ "$validator_config_file" -nt "$INVENTORY_FILE" ]; then +# Regenerate if main inventory missing, prepare inventory missing, or validator config is newer +_regen_inv=false +if [ ! -f "$INVENTORY_FILE" ]; then + _regen_inv=true +fi +if [ ! -f "$PREPARE_INVENTORY" ]; then + _regen_inv=true +fi +if [ -f "$validator_config_file" ] && [ -f "$INVENTORY_FILE" ] && [ "$validator_config_file" -nt "$INVENTORY_FILE" ]; then + _regen_inv=true +fi +if [ "$_regen_inv" = true ]; then echo "Generating Ansible inventory from validator-config.yaml..." "$scriptDir/generate-ansible-inventory.sh" "$validator_config_file" "$INVENTORY_FILE" fi -# Update inventory with SSH key file and user if provided +# prepare.yml: one play per physical host (deduped by IP). Deploy still uses full hosts.yml. +EFFECTIVE_INVENTORY="$INVENTORY_FILE" +if [ "$action" == "prepare" ] && [ -f "$PREPARE_INVENTORY" ]; then + EFFECTIVE_INVENTORY="$PREPARE_INVENTORY" +fi + +# Update inventory file(s) with SSH key file and user if provided if command -v yq &> /dev/null; then - # Derive the group list dynamically from the inventory so newly added clients - # (e.g. gean_nodes, lean_nodes) are automatically included without needing to - # update this hardcoded list every time a new client type is added. - all_groups=$(yq eval '.all.children | keys | .[]' "$INVENTORY_FILE" 2>/dev/null || echo "") - for group in $all_groups; do - # Get all hosts in this group - hosts=$(yq eval ".all.children.$group.hosts | keys | .[]" "$INVENTORY_FILE" 2>/dev/null || echo "") - for host in $hosts; do - # Only update if it's a remote host (has ansible_host but not ansible_connection: local) - connection=$(yq eval ".all.children.$group.hosts.$host.ansible_connection // \"\"" "$INVENTORY_FILE" 2>/dev/null) - if [ -z "$connection" ] || [ "$connection" != "local" ]; then - # Set SSH user (defaults to current user, or root if --useRoot flag is set) - yq eval -i ".all.children.$group.hosts.$host.ansible_user = \"$sshUser\"" "$INVENTORY_FILE" - - # Set SSH key file if provided - if [ -n "$sshKeyFile" ]; then - # Expand ~ to home directory if needed - if [[ "$sshKeyFile" == ~* ]]; then - sshKeyFile="${sshKeyFile/#\~/$HOME}" + _inv_files=("$INVENTORY_FILE") + [ -f "$PREPARE_INVENTORY" ] && _inv_files+=("$PREPARE_INVENTORY") + for _inv in "${_inv_files[@]}"; do + # Derive the group list dynamically from the inventory so newly added clients + # (e.g. gean_nodes, lean_nodes) are automatically included without needing to + # update this hardcoded list every time a new client type is added. + all_groups=$(yq eval '.all.children | keys | .[]' "$_inv" 2>/dev/null || echo "") + for group in $all_groups; do + # Get all hosts in this group + hosts=$(yq eval ".all.children.$group.hosts | keys | .[]" "$_inv" 2>/dev/null || echo "") + for host in $hosts; do + # Only update if it's a remote host (has ansible_host but not ansible_connection: local) + connection=$(yq eval ".all.children.$group.hosts.$host.ansible_connection // \"\"" "$_inv" 2>/dev/null) + if [ -z "$connection" ] || [ "$connection" != "local" ]; then + # Set SSH user (defaults to current user, or root if --useRoot flag is set) + yq eval -i ".all.children.$group.hosts.\"$host\".ansible_user = \"$sshUser\"" "$_inv" + + # Set SSH key file if provided + if [ -n "$sshKeyFile" ]; then + # Expand ~ to home directory if needed + if [[ "$sshKeyFile" == ~* ]]; then + sshKeyFile="${sshKeyFile/#\~/$HOME}" + fi + yq eval -i ".all.children.$group.hosts.\"$host\".ansible_ssh_private_key_file = \"$sshKeyFile\"" "$_inv" + echo "Setting SSH private key file for $host: $sshKeyFile" fi - yq eval -i ".all.children.$group.hosts.$host.ansible_ssh_private_key_file = \"$sshKeyFile\"" "$INVENTORY_FILE" - echo "Setting SSH private key file for $host: $sshKeyFile" fi - fi + done done done else @@ -116,9 +137,14 @@ if [ -n "$validatorConfig" ] && [ "$validatorConfig" != "genesis_bootnode" ]; th EXTRA_VARS="$EXTRA_VARS validator_config=$validatorConfig" fi -# Pass the full local path of the active validator config so deploy-nodes.yml -# can sync the correct file regardless of where it lives on disk. -EXTRA_VARS="$EXTRA_VARS local_validator_config_path=$validator_config_file" +# Pass the absolute path of the active validator config. ansible-playbook runs +# with cwd ansible/; lookup('file', ...) treats relative paths as relative to +# that directory, so a path like ansible-devnet/genesis/foo.yaml would break. +_local_vc_path="$validator_config_file" +if [[ "$_local_vc_path" != /* ]]; then + _local_vc_path="$scriptDir/$_local_vc_path" +fi +EXTRA_VARS="$EXTRA_VARS local_validator_config_path=$_local_vc_path" if [ -n "$coreDumps" ]; then EXTRA_VARS="$EXTRA_VARS enable_core_dumps=$coreDumps" @@ -164,7 +190,7 @@ fi # Build ansible-playbook command ANSIBLE_CMD="ansible-playbook" -ANSIBLE_CMD="$ANSIBLE_CMD -i $INVENTORY_FILE" +ANSIBLE_CMD="$ANSIBLE_CMD -i $EFFECTIVE_INVENTORY" ANSIBLE_CMD="$ANSIBLE_CMD $PLAYBOOK" ANSIBLE_CMD="$ANSIBLE_CMD -e \"$EXTRA_VARS\"" diff --git a/spin-node.sh b/spin-node.sh index d590950..30bc024 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -71,6 +71,13 @@ if [ "$deployment_mode" == "ansible" ] && ([ "$validatorConfig" == "genesis_boot echo "Using Ansible deployment: configDir=$configDir, validator config=$validator_config_file" fi +# --network is required for ansible; defaults to devnet-3 for local +if [ "$deployment_mode" == "ansible" ] && [ -z "$networkName" ]; then + echo "Error: --network is required for ansible deployments." + exit 1 +fi +networkName="${networkName:-devnet-3}" + # Set up logging if --logs flag is enabled if [ "$enableLogs" == "true" ]; then _log_dir="$scriptDir/tmp" @@ -103,29 +110,42 @@ fi # file with N nodes per client (same IP, unique incremented ports and keys). # This must run after configDir/validator_config_file are resolved so the # generated file lands in the correct genesis directory. +# +# Skip expansion when the file's attestation_committee_count already covers the +# requested subnet count (the config is already set up for that many subnets). +# --subnets takes precedence only when it exceeds the file's value (default 1). if [ -n "$subnets" ] && [ "$subnets" -ge 1 ] 2>/dev/null; then if ! [[ "$subnets" =~ ^[0-9]+$ ]] || [ "$subnets" -lt 1 ] || [ "$subnets" -gt 5 ]; then echo "Error: --subnets requires an integer between 1 and 5, got: $subnets" exit 1 fi - if ! command -v python3 &> /dev/null; then - echo "Error: python3 is required to generate the subnet config." - exit 1 + _file_ac=1 + if [ -f "$validator_config_file" ]; then + _file_ac=$(yq eval '.config.attestation_committee_count // 1' "$validator_config_file") fi - expanded_config="${configDir}/validator-config-subnets-${subnets}.yaml" - [ "$dryRun" == "true" ] && echo "[DRY RUN] Generating subnet config preview (no deployment will occur)" - echo "Generating subnet config ($subnets subnet(s) per client) → $expanded_config" + if [ "$_file_ac" -ge "$subnets" ] 2>/dev/null; then + echo "Config attestation_committee_count ($_file_ac) already covers --subnets $subnets; skipping subnet expansion." + else + if ! command -v python3 &> /dev/null; then + echo "Error: python3 is required to generate the subnet config." + exit 1 + fi - if ! python3 "$scriptDir/generate-subnet-config.py" \ - "$validator_config_file" "$subnets" "$expanded_config"; then - echo "❌ Failed to generate subnet config." - exit 1 - fi + expanded_config="${configDir}/validator-config-expanded.yaml" + [ "$dryRun" == "true" ] && echo "[DRY RUN] Generating subnet config preview (no deployment will occur)" + echo "Generating subnet config ($subnets subnet(s)) → $expanded_config" + + if ! python3 "$scriptDir/generate-subnet-config.py" \ + "$validator_config_file" "$subnets" "$expanded_config"; then + echo "❌ Failed to generate subnet config." + exit 1 + fi - validator_config_file="$expanded_config" - echo "Using expanded config: $validator_config_file" + validator_config_file="$expanded_config" + echo "Using expanded config: $validator_config_file" + fi fi # Handle --prepare mode: verify and install required software on all remote servers. @@ -154,8 +174,7 @@ if [ -n "$prepareMode" ] && [ "$prepareMode" == "true" ]; then [ -n "$dockerWithSudo" ] && ignored_flags+=("--dockerWithSudo") [ -n "$skipLeanpoint" ] && ignored_flags+=("--skip-leanpoint") [ -n "$skipNemo" ] && ignored_flags+=("--skip-nemo") - [ -n "$validatorConfig" ] && [ "$validatorConfig" != "genesis_bootnode" ] \ - && ignored_flags+=("--validatorConfig") + # --validatorConfig is allowed: inventory and prepare targets must match the chosen config. if [ ${#ignored_flags[@]} -gt 0 ]; then echo "" @@ -168,9 +187,12 @@ if [ -n "$prepareMode" ] && [ "$prepareMode" == "true" ]; then done echo "╠══════════════════════════════════════════════════════════════╣" echo "║ Allowed flags with --prepare: ║" + echo "║ • --validatorConfig ║" + echo "║ • --subnets N ║" echo "║ • --sshKey / --private-key ║" echo "║ • --useRoot ║" echo "║ • --deploymentMode ansible ║" + echo "║ • --network, --dry-run, --logs ║" echo "╚══════════════════════════════════════════════════════════════╝" echo "" exit 1 @@ -233,7 +255,7 @@ echo "Detected nodes: ${nodes[@]}" spin_nodes=() restart_with_checkpoint_sync=false -# Aggregator selection — one randomly chosen aggregator per subnet. +# Aggregator selection — one aggregator per subnet. # # Skipped entirely for --restart-client: restarting a single node must not # disturb the existing isAggregator assignments for the rest of the network. @@ -244,11 +266,47 @@ restart_with_checkpoint_sync=false # default to subnet 0 regardless of their name suffix. # # When --aggregator is specified, that node is used as the aggregator for -# its own subnet; all other subnets still get a random selection. - -# Helper: get the subnet index for a node from the config (defaults to 0). +# its own subnet; all other subnets still get a random selection (still +# excluding that node's client type from pools on other subnets). +# +# Default random mode (no --aggregator): aggregators are unique by CLIENT +# (prefix before the first '_', e.g. zeam from zeam_0). Example with 5 subnets: +# if zeam_* is chosen for subnet 0, no zeam_* node may be aggregator on +# subnets 1–4. If subnets outnumber distinct clients, the pool is exhausted +# and we fall back to unrestricted random with a warning. + +# Helper: get the subnet index for a node from the config. +# If the node has an explicit 'subnet' field, use it. +# Otherwise derive it from: validator_index % attestation_committee_count, +# where validator_index is the first validator assigned to this node by the +# genesis tool (cumulative sum of preceding nodes' 'count' fields). _node_subnet() { - yq eval ".validators[] | select(.name == \"$1\") | .subnet // 0" "$validator_config_file" + local _sn + _sn=$(yq eval ".validators[] | select(.name == \"$1\") | .subnet // \"\"" "$validator_config_file") + if [ -n "$_sn" ]; then + echo "$_sn" + return + fi + local _acc + _acc=$(yq eval '.config.attestation_committee_count // 1' "$validator_config_file") + local _vi=0 + local _line + while IFS=' ' read -r _name _count; do + if [ "$_name" == "$1" ]; then + echo $(( _vi % _acc )) + return + fi + _vi=$(( _vi + _count )) + done < <(yq eval '.validators[] | .name + " " + ((.count // 1) | tostring)' "$validator_config_file") + echo "0" +} + +# Helper: client type prefix (matches generate-subnet-config.py _client_name). +_client_prefix() { + case "$1" in + *_*) printf '%s\n' "${1%%_*}" ;; + *) printf '%s\n' "$1" ;; + esac } if [ -n "$restartClient" ]; then @@ -302,7 +360,9 @@ else # Select one aggregator per subnet and set the flag. # Priority: 1) --aggregator CLI flag 2) pre-existing isAggregator: true 3) random + # _used_agg_prefixes: client types already chosen (default random / preset / --aggregator). _aggregator_summary=() + _used_agg_prefixes=" " for _subnet_idx in "${_unique_subnets[@]}"; do _subnet_nodes=() for _node in "${nodes[@]}"; do @@ -324,16 +384,46 @@ else done if [[ "$_preset_valid" == "true" ]]; then _selected_agg="$_preset" + # Default mode: one client type at most once across subnets — drop conflicting presets. + if [ -z "$aggregatorNode" ]; then + _pp="$(_client_prefix "$_selected_agg")" + if [[ "$_used_agg_prefixes" == *" $_pp "* ]]; then + echo "Warning: preset aggregator '$_preset' (client $_pp) already aggregates another subnet; selecting randomly in subnet $_subnet_idx." >&2 + _selected_agg="" + fi + fi else # Preset node no longer exists — fall back to random and warn. echo "Warning: preset aggregator '$_preset' for subnet $_subnet_idx is not in the active node list; selecting randomly." >&2 - _selected_agg="${_subnet_nodes[$((RANDOM % ${#_subnet_nodes[@]}))]}" + _selected_agg="" fi - else - # 3. No preference set — pick randomly. - _selected_agg="${_subnet_nodes[$((RANDOM % ${#_subnet_nodes[@]}))]}" fi + # 3. Random (or preset fallback): prefer client types not yet used as aggregator. + if [ -z "$_selected_agg" ]; then + _eligible_aggs=() + for _n in "${_subnet_nodes[@]}"; do + _np="$(_client_prefix "$_n")" + case "$_used_agg_prefixes" in + *" $_np "*) : ;; + *) _eligible_aggs+=("$_n") ;; + esac + done + if [ ${#_eligible_aggs[@]} -eq 0 ] && [ ${#_subnet_nodes[@]} -gt 0 ]; then + echo "Warning: subnet $_subnet_idx — no unused client type left for aggregator (subnets > distinct clients?); picking among all nodes in this subnet." >&2 + _eligible_aggs=("${_subnet_nodes[@]}") + fi + if [ ${#_eligible_aggs[@]} -gt 0 ]; then + _selected_agg="${_eligible_aggs[$((RANDOM % ${#_eligible_aggs[@]}))]}" + else + echo "Error: subnet $_subnet_idx has no nodes to select an aggregator from." >&2 + exit 1 + fi + fi + + _sel_pref="$(_client_prefix "$_selected_agg")" + _used_agg_prefixes+="$_sel_pref " + if [ "$dryRun" != "true" ]; then yq eval -i "(.validators[] | select(.name == \"$_selected_agg\") | .isAggregator) = true" "$validator_config_file" fi @@ -364,7 +454,7 @@ else fi # end: aggregator selection (skipped for --restart-client) -# Print a prominent aggregator summary banner (only when aggregator selection ran). +# Print aggregator selection summary inline (quick confirmation during setup). if [ ${#_aggregator_summary[@]} -gt 0 ]; then echo "" echo "╔══════════════════════════════════════════════════════════════╗" @@ -377,6 +467,34 @@ if [ ${#_aggregator_summary[@]} -gt 0 ]; then echo "" fi +# Print deployment summary: subnet count and all clients per subnet. +_print_deployment_summary() { + if [ ${#_unique_subnets[@]} -gt 0 ]; then + echo "" + echo "╔══════════════════════════════════════════════════════════════╗" + echo "║ 📊 Deployment Summary ║" + echo "╠══════════════════════════════════════════════════════════════╣" + printf "║ %-60s║\n" "Subnets deployed: ${#_unique_subnets[@]}" + printf "║ %-60s║\n" "Total nodes: ${#nodes[@]}" + for _subnet_idx in "${_unique_subnets[@]}"; do + echo "╠══════════════════════════════════════════════════════════════╣" + printf "║ %-60s║\n" "Subnet $_subnet_idx:" + for _node in "${nodes[@]}"; do + if [[ "$(_node_subnet "$_node")" == "$_subnet_idx" ]]; then + _is_agg=$(yq eval ".validators[] | select(.name == \"$_node\") | .isAggregator" "$validator_config_file") + if [[ "$_is_agg" == "true" ]]; then + printf "║ %-58s║\n" "$_node (aggregator)" + else + printf "║ %-58s║\n" "$_node" + fi + fi + done + done + echo "╚══════════════════════════════════════════════════════════════╝" + echo "" + fi +} + # When --restart-client is specified, use it as the node list and enable checkpoint sync mode if [[ -n "$restartClient" ]]; then echo "Note: --restart-client is only used with --checkpoint-sync-url (default: https://leanpoint.leanroadmap.org/lean/v0/states/finalized)" @@ -645,7 +763,8 @@ if [ "$deployment_mode" == "ansible" ]; then fi fi - # Ansible deployment succeeded, exit normally + # Ansible deployment succeeded — print summary and exit. + _print_deployment_summary exit 0 fi @@ -835,8 +954,9 @@ for item in "${spin_nodes[@]}"; do echo "[DRY RUN] Would execute: $execCmd" pid=0 else + sed_remove_ansi='s/\x1b\[[0-9;]*[mJHG]//g' echo "$execCmd" - eval "$execCmd" & + eval "$execCmd" > >(tee >(sed -r "$sed_remove_ansi" > "$itemDataDir/stdout.log")) 2> >(tee >(sed -r "$sed_remove_ansi" > "$itemDataDir/stderr.log") >&2) & pid=$! fi spinned_pids+=($pid) @@ -936,6 +1056,8 @@ cleanup() { fi } +_print_deployment_summary + trap "echo exit signal received;cleanup" SIGINT SIGTERM echo -e "\n\nwaiting for nodes to exit" printf '%*s' $(tput cols) | tr ' ' '-'