From 2fc63d74c09e20b156c7e41a97fafaace7d3dfef Mon Sep 17 00:00:00 2001 From: Zhisheng Ye Date: Mon, 25 May 2026 12:59:41 +0800 Subject: [PATCH 1/4] Bootstrap Aether IPW simulator --- .github/workflows/ci.yml | 47 ++++ .gitignore | 4 + .gitmodules | 3 + .python-version | 1 + AGENTS.md | 52 ++++ README.md | 94 +++++++ docs/architecture.md | 43 +++ docs/formula-ledger.md | 60 ++++ docs/greensserve-ipw-plan.md | 16 ++ docs/mock-validation.md | 26 ++ docs/real-data-collection.md | 72 +++++ experiments/greensserve/baseline.yaml | 44 +++ experiments/greensserve/mock_measurement.yaml | 40 +++ experiments/greensserve/sglang_real.yaml | 50 ++++ .../0001-aether-ipw-metrics-scaffold.patch | 52 ++++ patches/sglang/v0.5.12/README.md | 25 ++ patches/sglang/v0.5.12/series | 1 + pyproject.toml | 39 +++ setup.py | 16 ++ src/aether/__init__.py | 3 + src/aether/cli.py | 98 +++++++ src/aether/config.py | 84 ++++++ src/aether/formulas.py | 91 ++++++ src/aether/measurement/__init__.py | 1 + src/aether/measurement/mock.py | 133 +++++++++ src/aether/measurement/sglang.py | 196 +++++++++++++ src/aether/results.py | 76 +++++ src/aether/simulator.py | 264 ++++++++++++++++++ tests/test_cli.py | 48 ++++ tests/test_config.py | 22 ++ tests/test_formulas.py | 18 ++ tests/test_results.py | 11 + tests/test_simulator.py | 21 ++ third_party/sglang | 1 + uv.lock | 153 ++++++++++ 35 files changed, 1905 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitmodules create mode 100644 .python-version create mode 100644 AGENTS.md create mode 100644 README.md create mode 100644 docs/architecture.md create mode 100644 docs/formula-ledger.md create mode 100644 docs/greensserve-ipw-plan.md create mode 100644 docs/mock-validation.md create mode 100644 docs/real-data-collection.md create mode 100644 experiments/greensserve/baseline.yaml create mode 100644 experiments/greensserve/mock_measurement.yaml create mode 100644 experiments/greensserve/sglang_real.yaml create mode 100644 patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch create mode 100644 patches/sglang/v0.5.12/README.md create mode 100644 patches/sglang/v0.5.12/series create mode 100644 pyproject.toml create mode 100644 setup.py create mode 100644 src/aether/__init__.py create mode 100644 src/aether/cli.py create mode 100644 src/aether/config.py create mode 100644 src/aether/formulas.py create mode 100644 src/aether/measurement/__init__.py create mode 100644 src/aether/measurement/mock.py create mode 100644 src/aether/measurement/sglang.py create mode 100644 src/aether/results.py create mode 100644 src/aether/simulator.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_config.py create mode 100644 tests/test_formulas.py create mode 100644 tests/test_results.py create mode 100644 tests/test_simulator.py create mode 160000 third_party/sglang create mode 100644 uv.lock diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ffac0ab --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: CI + +on: + push: + branches: [master, main] + pull_request: + +jobs: + cpu: + name: CPU checks + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Set up Python + run: uv python install 3.13 + + - name: Install dependencies + run: uv sync --group dev --group sglang + + - name: Verify Python + run: uv run python --version + + - name: Run tests + run: uv run pytest + + - name: Run simulation smoke + run: uv run aether simulate --config experiments/greensserve/baseline.yaml --out /tmp/aether-sim.csv + + - name: Run mock measurement smoke + run: uv run aether launch --backend mock --config experiments/greensserve/mock_measurement.yaml --out /tmp/aether-mock + + - name: Verify SGLang submodule pin + run: | + test "$(git -C third_party/sglang rev-parse HEAD)" = "127b9e3283f7c2a43234b852ff5c9f1796d53624" + + - name: Verify SGLang patch applies + run: | + git -C third_party/sglang apply --check ../../patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch diff --git a/.gitignore b/.gitignore index 83972fa..d2ecdcb 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,7 @@ ipython_config.py # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. # uv.lock +.uv-cache/ # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. @@ -206,6 +207,9 @@ tempCodeRunnerFile.py # Ruff stuff: .ruff_cache/ +# Aether experiment outputs +results/ + # PyPI configuration file .pypirc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..55c2bb3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "third_party/sglang"] + path = third_party/sglang + url = https://github.com/sgl-project/sglang.git diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..9ee5469 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,52 @@ +# AGENTS.md + +## Project + +Aether is a Python package and CLI for GreenServe intelligence-per-watt (IPW) +simulation and measurement. Use uv and Python 3.13. Keep the package installable with: + +```bash +uv python install 3.13 +uv sync --group dev +``` + +Run the CPU-safe checks with: + +```bash +uv run pytest +uv run aether simulate --config experiments/greensserve/baseline.yaml --out /tmp/aether-sim.csv +uv run aether launch --backend mock --config experiments/greensserve/mock_measurement.yaml --out /tmp/aether-mock +git -C third_party/sglang apply --check ../../patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch +``` + +## Network Proxy Preference + +When a download or dependency fetch appears stuck or blocked by network +restrictions, retry with these proxy variables before giving up: + +```bash +http_proxy=http://127.0.0.1:10808 +https_proxy=http://127.0.0.1:10808 +all_proxy=socks5://127.0.0.1:10808 +HTTP_PROXY=http://127.0.0.1:10808 +HTTPS_PROXY=http://127.0.0.1:10808 +ALL_PROXY=socks5://127.0.0.1:10808 +``` + +## Repo Rules + +- Keep real SGLang/GPU measurement code optional. Importing `aether` and running + tests must not require CUDA, NVML, SGLang, or a GPU. +- Use mock measurement as the correctness gate for launcher, collector, and + result-writer behavior. +- Keep normalized CSV/JSONL schemas shared across simulation, mock, and real + measurement paths. +- Store experiment outputs under `results/`; this directory is ignored. +- Keep SGLang work under `third_party/sglang` and patch files under + `patches/sglang/v0.5.12/`. +- SGLang patches in this bootstrap are metrics-only. Do not add scheduler, + recompute/swap, or DVFS behavior changes without a new design doc. +- SGLang `v0.5.12` does not fully install on local macOS arm64 in this + workspace because `sgl-deep-gemm==0.1.0` provides Linux wheels only. Use + macOS for patch-apply and Aether mock validation; use Linux GPU hosts for + real SGLang serving tests. diff --git a/README.md b/README.md new file mode 100644 index 0000000..a3009f7 --- /dev/null +++ b/README.md @@ -0,0 +1,94 @@ +# Aether + +Aether is an installable Python CLI for GreenServe-style intelligence-per-watt +(IPW) experiments. The current codebase is intentionally runnable on CPU: it +supports pure simulation and a deterministic mock measurement backend that +validates the same result schema used by future SGLang/GPU runs. + +The GitHub CI is CPU-only. It uses uv with Python 3.13, runs tests and smoke +commands, and verifies the SGLang metrics patch applies to the pinned submodule. + +Real GPU experiments are kept separate. The SGLang backend is present, but it +only runs when SGLang, CUDA/NVML, and a workload are available in the active +environment. + +Note: SGLang `v0.5.12` real serving is Linux/GPU-oriented. On this local macOS +arm64 machine, full install is blocked by upstream Linux-only `sgl-deep-gemm` +wheels; see `docs/real-data-collection.md` for the smoke-test result. + +## Install + +```bash +uv python install 3.13 +uv sync --group dev +``` + +After install: + +```bash +uv run aether doctor --config experiments/greensserve/baseline.yaml +uv run aether simulate --config experiments/greensserve/baseline.yaml --out results/sim.csv +uv run aether launch --backend mock --config experiments/greensserve/mock_measurement.yaml --out results/mock-run +uv run pytest +``` + +When published, the intended user install is: + +```bash +pip install aether +# or, with uv: +uv tool install aether +``` + +## CLI + +```bash +uv run aether simulate --config CONFIG.yaml --out results/sim.csv +uv run aether launch --backend mock --config CONFIG.yaml --out results/mock-run +uv run aether launch --backend sglang --config CONFIG.yaml --out results/real-run +uv run aether doctor --config CONFIG.yaml +uv run aether explain --config CONFIG.yaml +``` + +`simulate` expands YAML sweeps and writes one normalized CSV row per scenario. +`launch --backend mock` runs a deterministic CPU-only measurement fixture and +writes `summary.csv` plus `events.jsonl`. `launch --backend sglang` starts +`python -m sglang.launch_server` with the configured SGLang arguments, then +collects normalized measurement output if the real environment is available. + +## Config + +Examples live in `experiments/greensserve/`. + +- `baseline.yaml`: pure simulation sweep. +- `mock_measurement.yaml`: CPU-only launcher and collector validation. +- `sglang_real.yaml`: template for future SGLang/GPU measurement. + +The SGLang config accepts structured args: + +```yaml +sglang: + args: + model-path: meta-llama/Llama-3.1-8B-Instruct + host: 127.0.0.1 + port: 30000 + context-length: 8192 + enable-metrics: true + extra_args: + - --trust-remote-code +``` + +Booleans set to `true` are rendered as flags; `false` and `null` are omitted. +All other values are rendered as `--flag value`. + +## Real Data Collection + +Real collection is documented in `docs/real-data-collection.md`. In short: + +1. Initialize the SGLang submodule pinned to `v0.5.12`. +2. Apply the metrics-only patch series from `patches/sglang/v0.5.12/`. +3. Install patched SGLang in the active GPU environment. +4. Run `uv run aether launch --backend sglang --config experiments/greensserve/sglang_real.yaml --out results/real-run`. +5. Use `summary.csv`, `events.jsonl`, SGLang logs, and NVML samples for analysis. + +The current test suite does not require SGLang, CUDA, NVML, or a GPU. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..006fff7 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,43 @@ +# Aether Architecture + +Aether has three layers: + +1. Config and CLI: load YAML, render SGLang args, run commands, and write + normalized outputs. +2. Simulation: compute GreenServe/IPW metrics from reconstructed formulas and + scenario sweeps. +3. Measurement: collect normalized data from either deterministic CPU mock + fixtures or future real SGLang/GPU runs. + +Real measurement is isolated from simulation and mock validation. The SGLang +backend imports optional dependencies only inside the backend code path and +fails with setup guidance if SGLang, CUDA, or NVML are unavailable. + +## Data Flow + +```text +YAML config + -> config loader + -> simulate: sweep expander -> formula engine -> summary rows -> CSV + -> launch mock: synthetic events -> normalized summary/events -> CSV/JSONL + -> launch sglang: process launcher + samplers -> normalized summary/events +``` + +## Normalized Outputs + +All execution paths write the same summary fields where possible: + +- scenario identity: experiment, backend, scenario_id +- configuration: model, hardware, context length, compression ratio, routing + policy, scheduling policy +- capacity: KV bytes per token, effective KV bytes, max in-flight sequences +- timing: prefill seconds, decode seconds, elapsed seconds, TTFT/TBT estimates +- energy: joules, average watts, tokens/sec, tokens/watt, tokens/joule +- scheduling: swap bytes, recompute tokens, selected scheduling action +- quality: quality score and quality-normalized IPW + +## SGLang Integration + +The submodule is pinned to SGLang `v0.5.12`. The first patch series is +observability-only: it adds or documents hooks for power, token, KV, swap, and +preemption metrics without changing scheduling behavior. diff --git a/docs/formula-ledger.md b/docs/formula-ledger.md new file mode 100644 index 0000000..678b62b --- /dev/null +++ b/docs/formula-ledger.md @@ -0,0 +1,60 @@ +# GreenServe Formula Ledger + +Some inline formulas in the source Google Doc are embedded objects and were not +visible through text extraction. This ledger records the reconstructed formulas +implemented by Aether v0.1. + +## KV Cache + +```text +kv_bytes_per_token = 2 * layers * kv_heads * head_dim * kv_dtype_bytes +effective_kv_bytes = kv_bytes_per_token * compression_ratio + metadata_bytes_per_token +model_weight_bytes_per_gpu = model_weight_bytes / gpu_count +available_kv_vram_bytes = gpu_count * max(0, vram_bytes * gpu_memory_utilization - model_weight_bytes_per_gpu) +max_inflight = floor(available_kv_vram_bytes / (effective_kv_bytes * context_tokens)) +``` + +`compression_ratio=1.0` means uncompressed KV. Smaller values represent a +smaller retained memory footprint, for example `0.5` for half-size KV. + +## Energy And IPW + +```text +phase_energy_j = phase_seconds * phase_power_w +prefill_power_w = gpu_count * tdp_w * prefill_power_fraction +decode_power_w = gpu_count * tdp_w * decode_power_fraction +energy_j = prefill_energy_j + decode_energy_j + scheduling_energy_j +avg_power_w = energy_j / elapsed_seconds +tokens_per_second = generated_tokens / elapsed_seconds +tokens_per_watt = tokens_per_second / avg_power_w +tokens_per_joule = generated_tokens / energy_j +quality_normalized_ipw = quality_score * tokens_per_joule +``` + +Defaults from the GreenServe plan: + +- Prefill is compute-bound and uses `0.86 * TDP`. +- Decode is memory-bound and has a static floor of `0.43 * TDP`. + +## 1/W Context Law + +Aether models the 1/W law through KV capacity: + +```text +max_inflight is inversely proportional to context_tokens +effective_decode_throughput = decode_tps * min(1, max_inflight / target_concurrency) +``` + +As context grows, KV footprint grows linearly, maximum in-flight sequences fall, +and useful decode throughput per watt drops. + +## Recompute Vs Swap + +```text +risk_adjusted_cost = expected_cost + risk_weight * uncertainty +``` + +Recompute cost is modeled as another prefill/refill pass over preempted context. +Swap cost is modeled as PCIe transfer time over the request KV footprint plus +decode/static waiting power. The Bayesian scheduler chooses the lower +risk-adjusted cost. diff --git a/docs/greensserve-ipw-plan.md b/docs/greensserve-ipw-plan.md new file mode 100644 index 0000000..0ffd573 --- /dev/null +++ b/docs/greensserve-ipw-plan.md @@ -0,0 +1,16 @@ +# GreenServe IPW Plan Summary + +Aether implements the first software layer for the GreenServe research plan: +modeling and measuring intelligence per watt for LLM serving systems. + +The relevant system components are: + +- context-length-aware routing pools inspired by FleetOpt +- KV-cache compression and its effect on in-flight capacity +- compute-bound prefill and memory-bound decode phase asymmetry +- recompute-vs-swap scheduling decisions under VRAM pressure +- optional SGLang metrics collection for future real experiments + +The initial implementation is deliberately simulation-first. It lets the team +debug formulas, scenario schemas, CLI behavior, and normalized output formats on +CPU before running expensive GPU experiments. diff --git a/docs/mock-validation.md b/docs/mock-validation.md new file mode 100644 index 0000000..1cff703 --- /dev/null +++ b/docs/mock-validation.md @@ -0,0 +1,26 @@ +# Mock Validation + +The mock backend is the CPU-only correctness gate for Aether. It does not start +SGLang and does not require GPU libraries. + +Run: + +```bash +aether launch --backend mock --config experiments/greensserve/mock_measurement.yaml --out /tmp/aether-mock +``` + +Expected outputs: + +- `/tmp/aether-mock/summary.csv` +- `/tmp/aether-mock/events.jsonl` + +The mock backend emits deterministic event streams: + +- power samples +- token events +- KV cache events +- scheduler events for recompute and swap +- latency samples + +Because the events are deterministic, tests can compare stable CSV columns and +known metric ranges without relying on hardware. diff --git a/docs/real-data-collection.md b/docs/real-data-collection.md new file mode 100644 index 0000000..55789c2 --- /dev/null +++ b/docs/real-data-collection.md @@ -0,0 +1,72 @@ +# Real SGLang Data Collection + +Real measurement is separate from CPU validation. Use it only on a machine with +NVIDIA GPUs, working drivers, CUDA, NVML, and an installed SGLang environment. + +## Prepare SGLang + +```bash +uv sync --group dev --group sglang +git submodule update --init --recursive third_party/sglang +cd third_party/sglang +git checkout v0.5.12 +git apply --check ../../patches/sglang/v0.5.12/*.patch +git apply ../../patches/sglang/v0.5.12/*.patch +uv pip install -e "python[all]" +``` + +The bootstrap patch series is metrics-only. It must not change scheduling, +DVFS, or recompute/swap behavior. + +## macOS Local Smoke Result + +SGLang `v0.5.12` is not currently installable as a real local macOS arm64 +serving stack in this workspace: + +- Full dependency install fails because `sglang==0.5.12` depends on + `sgl-deep-gemm==0.1.0`, and uv reports wheels only for + `manylinux2014_aarch64` and `manylinux2014_x86_64`. +- A no-dependency editable smoke install from a patched SGLang worktree reaches + SGLang's build step, then fails because `rustc` is not installed locally and + SGLang builds a Rust extension. + +What is verified on macOS: + +```bash +git -C third_party/sglang worktree add /private/tmp/aether-sglang-v0.5.12-test v0.5.12 +git -C /private/tmp/aether-sglang-v0.5.12-test apply /path/to/Aether/patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch +``` + +The patch applies cleanly and creates `AETHER_IPW_METRICS.md`. A real SGLang +server run should be tested on Linux with the SGLang-supported GPU stack. + +## Run Aether + +Edit `experiments/greensserve/sglang_real.yaml` for the model, port, context +length, workload, and SGLang arguments. + +```bash +uv run aether doctor --config experiments/greensserve/sglang_real.yaml +uv run aether launch --backend sglang --config experiments/greensserve/sglang_real.yaml --out results/real-run +``` + +The SGLang backend will: + +1. render configured SGLang args +2. launch `python -m sglang.launch_server` +3. wait for `/health` +4. sample NVML power when `measurement.nvml_enabled` is true +5. optionally run the configured workload command +6. write normalized `summary.csv` and `events.jsonl` + +## Data To Record + +- NVML power samples at 100 Hz or higher +- TTFT, TBT, and end-to-end latency +- prompt and generation token counts +- KV cache usage +- swap, recompute, and preemption counters +- SGLang server logs and metrics endpoint snapshots + +Keep raw outputs alongside normalized summaries so future formula changes can +recompute metrics from the source data. diff --git a/experiments/greensserve/baseline.yaml b/experiments/greensserve/baseline.yaml new file mode 100644 index 0000000..0db442a --- /dev/null +++ b/experiments/greensserve/baseline.yaml @@ -0,0 +1,44 @@ +experiment: + name: greensserve-baseline + tags: [simulation, ipw, greensserve] + +model: + name: llama-3.1-70b + layers: 80 + kv_heads: 8 + head_dim: 128 + kv_dtype_bytes: 2 + weight_gb: 140 + quality_score: 0.92 + +workload: + prompt_tokens: 4096 + generated_tokens: 256 + target_concurrency: 256 + preemption_probability: 0.05 + ttft_slo_ms: 2000 + tbt_slo_ms: 200 + +simulation: + hardware_profiles: + - name: h100-sxm-80gb + gpu_count: 8 + tdp_w: 700 + vram_gb: 80 + gpu_memory_utilization: 0.9 + hbm_bandwidth_gbps: 3350 + pcie_bandwidth_gbps: 64 + prefill_tps_per_gpu: 9000 + decode_tps_per_gpu: 320 + routing_pools: + - name: short-context-pool + max_context_tokens: 8192 + gpu_count: 4 + - name: long-context-pool + max_context_tokens: 65536 + gpu_count: 4 + sweeps: + context_lengths: [4096, 8192, 32768, 65536] + kv_compression_ratios: [1.0, 0.5, 0.25] + routing_policies: [single_pool, fleetopt] + scheduling_policies: [none, recompute, swap, bayesian] diff --git a/experiments/greensserve/mock_measurement.yaml b/experiments/greensserve/mock_measurement.yaml new file mode 100644 index 0000000..f5cb354 --- /dev/null +++ b/experiments/greensserve/mock_measurement.yaml @@ -0,0 +1,40 @@ +experiment: + name: greensserve-mock + tags: [mock, cpu, measurement] + +model: + name: mock-llm + layers: 32 + kv_heads: 8 + head_dim: 128 + kv_dtype_bytes: 2 + weight_gb: 16 + quality_score: 0.88 + +workload: + prompt_tokens: 2048 + generated_tokens: 128 + target_concurrency: 16 + preemption_probability: 0.1 + +measurement: + duration_seconds: 5 + sample_hz: 10 + mock_seed: 7 + request_count: 8 + +simulation: + hardware_profiles: + - name: mock-gpu + gpu_count: 1 + tdp_w: 300 + vram_gb: 24 + gpu_memory_utilization: 0.9 + pcie_bandwidth_gbps: 32 + prefill_tps_per_gpu: 4000 + decode_tps_per_gpu: 220 + sweeps: + context_lengths: [2048] + kv_compression_ratios: [1.0] + routing_policies: [single_pool] + scheduling_policies: [bayesian] diff --git a/experiments/greensserve/sglang_real.yaml b/experiments/greensserve/sglang_real.yaml new file mode 100644 index 0000000..cc1492f --- /dev/null +++ b/experiments/greensserve/sglang_real.yaml @@ -0,0 +1,50 @@ +experiment: + name: greensserve-sglang-real + tags: [sglang, gpu, real-measurement] + +model: + name: llama-3.1-8b-instruct + layers: 32 + kv_heads: 8 + head_dim: 128 + kv_dtype_bytes: 2 + weight_gb: 16 + quality_score: 0.90 + +sglang: + args: + model-path: meta-llama/Llama-3.1-8B-Instruct + host: 127.0.0.1 + port: 30000 + context-length: 8192 + enable-metrics: true + extra_args: [] + +measurement: + duration_seconds: 60 + sample_hz: 100 + nvml_enabled: true + health_timeout_seconds: 120 + workload_command: null + +workload: + prompt_tokens: 4096 + generated_tokens: 256 + target_concurrency: 32 + preemption_probability: 0.0 + +simulation: + hardware_profiles: + - name: real-gpu-template + gpu_count: 1 + tdp_w: 700 + vram_gb: 80 + gpu_memory_utilization: 0.9 + pcie_bandwidth_gbps: 64 + prefill_tps_per_gpu: 9000 + decode_tps_per_gpu: 320 + sweeps: + context_lengths: [8192] + kv_compression_ratios: [1.0] + routing_policies: [single_pool] + scheduling_policies: [none] diff --git a/patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch b/patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch new file mode 100644 index 0000000..021117c --- /dev/null +++ b/patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch @@ -0,0 +1,52 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Aether Contributors +Date: Mon, 25 May 2026 00:00:00 +0000 +Subject: [PATCH] docs: add Aether IPW metrics scaffold + +--- + AETHER_IPW_METRICS.md | 32 ++++++++++++++++++++++++++++++++ + 1 file changed, 32 insertions(+) + create mode 100644 AETHER_IPW_METRICS.md + +diff --git a/AETHER_IPW_METRICS.md b/AETHER_IPW_METRICS.md +new file mode 100644 +index 0000000000..1111111111 +--- /dev/null ++++ b/AETHER_IPW_METRICS.md +@@ -0,0 +1,32 @@ ++# Aether IPW Metrics Scaffold ++ ++This file is added by the Aether metrics-only patch scaffold. It does not ++change SGLang behavior. ++ ++Future metrics patches should expose the following data without changing ++scheduler, DVFS, or request execution behavior: ++ ++- per-GPU NVML power samples, clocks, utilization, and memory usage ++- per-request prompt token count, generated token count, TTFT, TBT, and E2E ++ latency ++- KV cache allocation, free, compression ratio, and effective bytes per token ++- preemption events, recompute events, swap in/out events, and transferred bytes ++- SGLang process metadata: model, tensor parallel size, max context, and port ++ ++Aether normalizes these values into: ++ ++- `summary.csv` ++- `events.jsonl` ++ ++The intended first real implementation points are: ++ ++1. server launch args for enabling Aether metrics ++2. metrics registry counters and gauges ++3. scheduler event hooks ++4. KV cache pool event hooks ++5. JSONL export for per-request and per-iteration events ++ ++Patch constraints: ++ ++- no scheduling behavior changes ++- no DVFS behavior changes ++- no extra dependencies in SGLang's default import path ++- metrics disabled by default +-- +2.39.0 diff --git a/patches/sglang/v0.5.12/README.md b/patches/sglang/v0.5.12/README.md new file mode 100644 index 0000000..49703d2 --- /dev/null +++ b/patches/sglang/v0.5.12/README.md @@ -0,0 +1,25 @@ +# Aether SGLang Patch Series For v0.5.12 + +This directory contains the first Aether patch scaffold for SGLang `v0.5.12`. +The bootstrap goal is metrics only: + +- power and clock samples +- request token counts +- TTFT/TBT/end-to-end latency markers +- KV cache usage +- preemption, recompute, and swap counters where SGLang exposes them + +No patch in this series should alter scheduling behavior, DVFS behavior, model +execution, or request routing. + +Apply from the repo root after initializing the submodule: + +```bash +git submodule update --init --recursive third_party/sglang +cd third_party/sglang +git checkout v0.5.12 +git apply --check ../../patches/sglang/v0.5.12/*.patch +``` + +`0001-aether-ipw-metrics-scaffold.patch` is intentionally minimal until the +exact SGLang metrics insertion points are selected in a follow-up patch. diff --git a/patches/sglang/v0.5.12/series b/patches/sglang/v0.5.12/series new file mode 100644 index 0000000..96e163d --- /dev/null +++ b/patches/sglang/v0.5.12/series @@ -0,0 +1 @@ +0001-aether-ipw-metrics-scaffold.patch diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b924b75 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "aether" +version = "0.1.0" +description = "Aether: simulation and measurement tooling for GreenServe intelligence-per-watt experiments." +readme = "README.md" +requires-python = ">=3.13" +license = {text = "Apache-2.0"} +authors = [ + {name = "Aether Contributors", email = "aether@yezhisheng.me"} +] +dependencies = [ + "PyYAML>=5.4" +] + +[project.optional-dependencies] +sglang = [ + "pynvml>=11.5" +] + +[dependency-groups] +dev = [ + "pytest>=8.3" +] +sglang = [ + "pynvml>=11.5" +] + +[project.scripts] +aether = "aether.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..246663b --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +from setuptools import find_packages, setup + + +setup( + name="aether", + version="0.1.0", + description="Aether: simulation and measurement tooling for GreenServe intelligence-per-watt experiments.", + author="Aether Contributors", + author_email="aether@yezhisheng.me", + package_dir={"": "src"}, + packages=find_packages("src"), + install_requires=["PyYAML>=5.4"], + extras_require={"dev": ["pytest>=8.3"], "sglang": ["pynvml>=11.5"]}, + entry_points={"console_scripts": ["aether=aether.cli:main"]}, + python_requires=">=3.13", +) diff --git a/src/aether/__init__.py b/src/aether/__init__.py new file mode 100644 index 0000000..83017eb --- /dev/null +++ b/src/aether/__init__.py @@ -0,0 +1,3 @@ +"""Aether simulation and measurement tooling.""" + +__version__ = "0.1.0" diff --git a/src/aether/cli.py b/src/aether/cli.py new file mode 100644 index 0000000..e144ade --- /dev/null +++ b/src/aether/cli.py @@ -0,0 +1,98 @@ +"""Aether command line interface.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import List, Optional + +from .config import ConfigError, load_config, render_sglang_args +from .measurement.mock import run_mock +from .measurement.sglang import SGLangMeasurementError, run_sglang +from .results import write_csv +from .simulator import simulate + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="aether", description="Aether IPW simulation and measurement CLI") + subcommands = parser.add_subparsers(dest="command", required=True) + + simulate_parser = subcommands.add_parser("simulate", help="run an offline IPW scenario sweep") + simulate_parser.add_argument("--config", required=True) + simulate_parser.add_argument("--out", required=True) + + launch_parser = subcommands.add_parser("launch", help="launch a measurement backend") + launch_parser.add_argument("--backend", choices=["mock", "sglang"], required=True) + launch_parser.add_argument("--config", required=True) + launch_parser.add_argument("--out", required=True) + + doctor_parser = subcommands.add_parser("doctor", help="validate config and print environment guidance") + doctor_parser.add_argument("--config", required=True) + + explain_parser = subcommands.add_parser("explain", help="print expanded scenarios and rendered SGLang args") + explain_parser.add_argument("--config", required=True) + return parser + + +def _cmd_simulate(args: argparse.Namespace) -> int: + config = load_config(args.config) + rows = simulate(config) + write_csv(rows, args.out) + print("wrote %s rows to %s" % (len(rows), args.out)) + return 0 + + +def _cmd_launch(args: argparse.Namespace) -> int: + config = load_config(args.config) + if args.backend == "mock": + row, events = run_mock(config, args.out) + else: + row, events = run_sglang(config, args.out) + print("wrote summary.csv and events.jsonl to %s" % args.out) + print("backend=%s scenario_id=%s events=%s" % (args.backend, row.get("scenario_id"), len(events))) + return 0 + + +def _cmd_doctor(args: argparse.Namespace) -> int: + config = load_config(args.config) + rows = simulate(config) + print("config: %s" % Path(args.config)) + print("simulation_scenarios: %s" % len(rows)) + print("sglang_args: %s" % " ".join(render_sglang_args(config))) + print("cpu_safe: true") + print("real_sglang_requires: SGLang, CUDA, NVML, GPU, and a workload") + return 0 + + +def _cmd_explain(args: argparse.Namespace) -> int: + config = load_config(args.config) + payload = { + "sglang_args": render_sglang_args(config), + "scenarios": simulate(config), + } + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + + +def main(argv: Optional[List[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + if args.command == "simulate": + return _cmd_simulate(args) + if args.command == "launch": + return _cmd_launch(args) + if args.command == "doctor": + return _cmd_doctor(args) + if args.command == "explain": + return _cmd_explain(args) + except (ConfigError, SGLangMeasurementError, ValueError) as exc: + print("aether: %s" % exc, file=sys.stderr) + return 2 + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/aether/config.py b/src/aether/config.py new file mode 100644 index 0000000..2e9d23c --- /dev/null +++ b/src/aether/config.py @@ -0,0 +1,84 @@ +"""Configuration loading and SGLang argument rendering.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Iterable, List, Mapping + +import yaml + + +class ConfigError(ValueError): + """Raised when an Aether YAML config is invalid.""" + + +def load_config(path: str) -> Dict[str, Any]: + config_path = Path(path) + if not config_path.exists(): + raise ConfigError("config file does not exist: %s" % config_path) + with config_path.open("r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + if not isinstance(data, dict): + raise ConfigError("config root must be a mapping") + return data + + +def require_mapping(data: Mapping[str, Any], key: str) -> Dict[str, Any]: + value = data.get(key) + if value is None: + return {} + if not isinstance(value, dict): + raise ConfigError("%s must be a mapping" % key) + return dict(value) + + +def as_list(value: Any, default: Iterable[Any]) -> List[Any]: + if value is None: + return list(default) + if isinstance(value, list): + return value + return [value] + + +def render_flag_name(name: str) -> str: + return "--" + name.replace("_", "-") + + +def render_cli_args(args: Mapping[str, Any]) -> List[str]: + rendered = [] + for key in sorted(args): + value = args[key] + flag = render_flag_name(str(key)) + if value is None or value is False: + continue + if value is True: + rendered.append(flag) + continue + if isinstance(value, (list, tuple)): + for item in value: + rendered.extend([flag, str(item)]) + continue + if isinstance(value, dict): + raise ConfigError("nested SGLang arg %s is not supported" % key) + rendered.extend([flag, str(value)]) + return rendered + + +def render_sglang_args(config: Mapping[str, Any]) -> List[str]: + sglang = require_mapping(config, "sglang") + args = sglang.get("args", {}) + if args is None: + args = {} + if not isinstance(args, dict): + raise ConfigError("sglang.args must be a mapping") + extra_args = sglang.get("extra_args", []) + if extra_args is None: + extra_args = [] + if not isinstance(extra_args, list): + raise ConfigError("sglang.extra_args must be a list") + return render_cli_args(args) + [str(item) for item in extra_args] + + +def experiment_name(config: Mapping[str, Any]) -> str: + experiment = require_mapping(config, "experiment") + return str(experiment.get("name", "aether-experiment")) diff --git a/src/aether/formulas.py b/src/aether/formulas.py new file mode 100644 index 0000000..29ee5f0 --- /dev/null +++ b/src/aether/formulas.py @@ -0,0 +1,91 @@ +"""Reconstructed GreenServe/IPW formulas.""" + +from __future__ import annotations + +import math + + +BYTES_PER_GB = 1024 ** 3 + + +def kv_bytes_per_token(layers: int, kv_heads: int, head_dim: int, kv_dtype_bytes: int) -> int: + return int(2 * layers * kv_heads * head_dim * kv_dtype_bytes) + + +def effective_kv_bytes( + kv_bytes: float, + compression_ratio: float, + metadata_bytes_per_token: float = 0.0, +) -> float: + if compression_ratio <= 0: + raise ValueError("compression_ratio must be positive") + return kv_bytes * compression_ratio + metadata_bytes_per_token + + +def available_kv_vram_bytes( + vram_gb: float, + model_weight_gb: float, + gpu_memory_utilization: float = 0.9, +) -> float: + if not 0 < gpu_memory_utilization <= 1: + raise ValueError("gpu_memory_utilization must be in (0, 1]") + return max(0.0, (vram_gb * gpu_memory_utilization - model_weight_gb) * BYTES_PER_GB) + + +def max_inflight_sequences( + available_vram_bytes: float, + effective_kv_bytes_per_token: float, + context_tokens: int, +) -> int: + if effective_kv_bytes_per_token <= 0 or context_tokens <= 0: + return 0 + return int(math.floor(available_vram_bytes / (effective_kv_bytes_per_token * context_tokens))) + + +def phase_energy_j(seconds: float, power_w: float) -> float: + return max(0.0, seconds) * max(0.0, power_w) + + +def safe_div(numerator: float, denominator: float) -> float: + if denominator == 0: + return 0.0 + return numerator / denominator + + +def risk_adjusted_cost(expected_cost: float, uncertainty: float, risk_weight: float) -> float: + return expected_cost + risk_weight * uncertainty + + +def recompute_cost( + context_tokens: int, + prefill_tps: float, + tdp_w: float, + gpu_count: int, + prefill_power_fraction: float = 0.86, + uncertainty_fraction: float = 0.10, +) -> dict: + seconds = safe_div(context_tokens, prefill_tps) + energy = phase_energy_j(seconds, gpu_count * tdp_w * prefill_power_fraction) + return { + "seconds": seconds, + "energy_j": energy, + "uncertainty_j": energy * uncertainty_fraction, + } + + +def swap_cost( + kv_bytes: float, + pcie_bandwidth_gbps: float, + tdp_w: float, + gpu_count: int, + decode_power_fraction: float = 0.43, + uncertainty_fraction: float = 0.20, +) -> dict: + bandwidth_bytes_s = max(1.0, pcie_bandwidth_gbps * 1_000_000_000.0) + seconds = kv_bytes / bandwidth_bytes_s + energy = phase_energy_j(seconds, gpu_count * tdp_w * decode_power_fraction) + return { + "seconds": seconds, + "energy_j": energy, + "uncertainty_j": energy * uncertainty_fraction, + } diff --git a/src/aether/measurement/__init__.py b/src/aether/measurement/__init__.py new file mode 100644 index 0000000..dd94153 --- /dev/null +++ b/src/aether/measurement/__init__.py @@ -0,0 +1 @@ +"""Measurement backends for Aether.""" diff --git a/src/aether/measurement/mock.py b/src/aether/measurement/mock.py new file mode 100644 index 0000000..6e83083 --- /dev/null +++ b/src/aether/measurement/mock.py @@ -0,0 +1,133 @@ +"""Deterministic CPU-only measurement backend.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, List, Mapping, Tuple + +from .. import formulas +from ..config import experiment_name +from ..results import write_csv, write_jsonl + + +def _measurement(config: Mapping[str, Any]) -> Dict[str, Any]: + measurement = dict(config.get("measurement", {}) or {}) + measurement.setdefault("duration_seconds", 5) + measurement.setdefault("sample_hz", 10) + measurement.setdefault("request_count", 8) + return measurement + + +def _model(config: Mapping[str, Any]) -> Dict[str, Any]: + model = dict(config.get("model", {}) or {}) + model.setdefault("name", "mock-model") + model.setdefault("layers", 32) + model.setdefault("kv_heads", 8) + model.setdefault("head_dim", 128) + model.setdefault("kv_dtype_bytes", 2) + model.setdefault("quality_score", 1.0) + return model + + +def _workload(config: Mapping[str, Any]) -> Dict[str, Any]: + workload = dict(config.get("workload", {}) or {}) + workload.setdefault("prompt_tokens", 2048) + workload.setdefault("generated_tokens", 128) + workload.setdefault("target_concurrency", 8) + return workload + + +def run_mock(config: Mapping[str, Any], out_dir: str) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: + measurement = _measurement(config) + model = _model(config) + workload = _workload(config) + exp_name = experiment_name(config) + duration = float(measurement["duration_seconds"]) + sample_hz = int(measurement["sample_hz"]) + request_count = int(measurement["request_count"]) + prompt_tokens = int(workload["prompt_tokens"]) + generated_per_request = int(workload["generated_tokens"]) + generated_tokens = request_count * generated_per_request + kv_bytes = formulas.kv_bytes_per_token( + int(model["layers"]), + int(model["kv_heads"]), + int(model["head_dim"]), + int(model["kv_dtype_bytes"]), + ) + + events = [] + samples = max(1, int(duration * sample_hz)) + power_values = [] + for index in range(samples): + ts = round(index / float(sample_hz), 6) + power_w = 225.0 + float((index * 7) % 19) + power_values.append(power_w) + events.append({"type": "power", "time_s": ts, "power_w": power_w, "backend": "mock"}) + + for request_id in range(request_count): + token_count = generated_per_request + events.append( + { + "type": "request", + "request_id": request_id, + "prompt_tokens": prompt_tokens, + "generated_tokens": token_count, + "ttft_ms": 35.0 + request_id, + "tbt_ms": 12.0 + (request_id % 3), + } + ) + + events.extend( + [ + {"type": "kv", "allocated_bytes": kv_bytes * prompt_tokens * request_count, "kv_bytes_per_token": kv_bytes}, + {"type": "scheduler", "action": "recompute", "request_id": 0, "tokens": prompt_tokens // 2}, + {"type": "scheduler", "action": "swap", "request_id": 1, "bytes": kv_bytes * prompt_tokens}, + ] + ) + + avg_power = sum(power_values) / len(power_values) + energy = avg_power * duration + tokens_per_second = formulas.safe_div(generated_tokens, duration) + tokens_per_joule = formulas.safe_div(generated_tokens, energy) + quality = float(model.get("quality_score", 1.0)) + row = { + "experiment": exp_name, + "backend": "mock", + "scenario_id": exp_name + "-mock", + "model": model.get("name", "mock-model"), + "hardware": "mock-cpu", + "routing_policy": "mock", + "routing_pool": "mock", + "scheduling_policy": "mock", + "scheduling_action": "mixed", + "context_tokens": prompt_tokens, + "prompt_tokens": prompt_tokens, + "generated_tokens": generated_tokens, + "target_concurrency": workload.get("target_concurrency", request_count), + "active_sequences": request_count, + "kv_compression_ratio": 1.0, + "kv_bytes_per_token": kv_bytes, + "effective_kv_bytes_per_token": kv_bytes, + "max_inflight_sequences": request_count, + "prefill_seconds": 0.035, + "decode_seconds": round(max(0.0, duration - 0.035), 6), + "elapsed_seconds": duration, + "energy_j": round(energy, 6), + "avg_power_w": round(avg_power, 6), + "tokens_per_second": round(tokens_per_second, 6), + "tokens_per_watt": round(formulas.safe_div(tokens_per_second, avg_power), 9), + "tokens_per_joule": round(tokens_per_joule, 9), + "quality_score": quality, + "quality_normalized_ipw": round(quality * tokens_per_joule, 9), + "swap_bytes": kv_bytes * prompt_tokens, + "recompute_tokens": prompt_tokens // 2, + "ttft_ms": 38.5, + "tbt_ms": 13.0, + "slo_violations": 0, + } + + output_dir = Path(out_dir) + output_dir.mkdir(parents=True, exist_ok=True) + write_csv([row], str(output_dir / "summary.csv")) + write_jsonl(events, str(output_dir / "events.jsonl")) + return row, events diff --git a/src/aether/measurement/sglang.py b/src/aether/measurement/sglang.py new file mode 100644 index 0000000..9c14327 --- /dev/null +++ b/src/aether/measurement/sglang.py @@ -0,0 +1,196 @@ +"""Real SGLang launcher and measurement backend. + +This module is intentionally optional at runtime. It imports GPU-specific +libraries only inside functions so CPU simulation and tests remain clean. +""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Tuple +from urllib.error import URLError +from urllib.request import urlopen + +from ..config import render_sglang_args +from ..results import write_csv, write_jsonl + + +class SGLangMeasurementError(RuntimeError): + """Raised when the real SGLang backend cannot run.""" + + +def _require_sglang() -> None: + if importlib.util.find_spec("sglang") is None: + raise SGLangMeasurementError( + "SGLang is not installed in this environment. Install patched SGLang " + "from third_party/sglang before running --backend sglang." + ) + + +def _health_url(config: Mapping[str, Any]) -> str: + args = dict((config.get("sglang", {}) or {}).get("args", {}) or {}) + host = str(args.get("host", "127.0.0.1")) + port = str(args.get("port", "30000")) + return "http://%s:%s/health" % (host, port) + + +def _wait_for_health(url: str, timeout_s: float) -> bool: + deadline = time.time() + timeout_s + while time.time() < deadline: + try: + with urlopen(url, timeout=2) as response: + if 200 <= response.status < 500: + return True + except URLError: + time.sleep(1) + return False + + +class _NvmlSampler: + def __init__(self, sample_hz: int, events: List[Dict[str, Any]]): + self.sample_hz = max(1, int(sample_hz)) + self.events = events + self.stop_event = threading.Event() + self.thread: Optional[threading.Thread] = None + self.error: Optional[BaseException] = None + + def start(self) -> None: + self.thread = threading.Thread(target=self._run, name="aether-nvml-sampler") + self.thread.daemon = True + self.thread.start() + time.sleep(0.05) + if self.error is not None: + raise SGLangMeasurementError(str(self.error)) + + def stop(self) -> None: + self.stop_event.set() + if self.thread is not None: + self.thread.join(timeout=5) + if self.error is not None: + raise SGLangMeasurementError(str(self.error)) + + def _run(self) -> None: + try: + import pynvml # type: ignore + + pynvml.nvmlInit() + device_count = pynvml.nvmlDeviceGetCount() + interval = 1.0 / float(self.sample_hz) + start = time.time() + while not self.stop_event.is_set(): + ts = round(time.time() - start, 6) + for gpu_id in range(device_count): + handle = pynvml.nvmlDeviceGetHandleByIndex(gpu_id) + power_w = pynvml.nvmlDeviceGetPowerUsage(handle) / 1000.0 + memory = pynvml.nvmlDeviceGetMemoryInfo(handle) + self.events.append( + { + "type": "power", + "backend": "sglang", + "time_s": ts, + "gpu_id": gpu_id, + "power_w": power_w, + "memory_used_bytes": int(memory.used), + "memory_total_bytes": int(memory.total), + } + ) + self.stop_event.wait(interval) + pynvml.nvmlShutdown() + except BaseException as exc: # pragma: no cover - requires GPU/NVML + self.error = exc + + +def _power_summary(events: List[Dict[str, Any]], fallback_duration: float) -> Tuple[float, float, float]: + by_ts: Dict[float, float] = {} + for event in events: + if event.get("type") == "power": + by_ts.setdefault(float(event.get("time_s", 0.0)), 0.0) + by_ts[float(event.get("time_s", 0.0))] += float(event.get("power_w", 0.0)) + if not by_ts: + return fallback_duration, 0.0, 0.0 + elapsed = max(by_ts) if max(by_ts) > 0 else fallback_duration + avg_power = sum(by_ts.values()) / len(by_ts) + return elapsed, avg_power, avg_power * elapsed + + +def run_sglang(config: Mapping[str, Any], out_dir: str) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: + _require_sglang() + output_dir = Path(out_dir) + output_dir.mkdir(parents=True, exist_ok=True) + log_path = output_dir / "sglang.log" + measurement = dict(config.get("measurement", {}) or {}) + duration = float(measurement.get("duration_seconds", 60)) + timeout = float(measurement.get("health_timeout_seconds", 120)) + sample_hz = int(measurement.get("sample_hz", 100)) + nvml_enabled = bool(measurement.get("nvml_enabled", True)) + args = render_sglang_args(config) + command = [sys.executable, "-m", "sglang.launch_server"] + args + + events: List[Dict[str, Any]] = [{"type": "launch", "command": command}] + sampler = _NvmlSampler(sample_hz, events) if nvml_enabled else None + with log_path.open("w", encoding="utf-8") as log_handle: + process = subprocess.Popen(command, stdout=log_handle, stderr=subprocess.STDOUT) + try: + health_url = _health_url(config) + if not _wait_for_health(health_url, timeout): + raise SGLangMeasurementError("SGLang did not become healthy at %s" % health_url) + events.append({"type": "health", "url": health_url, "ok": True}) + if sampler is not None: + sampler.start() + workload_command = measurement.get("workload_command") + if workload_command: + if not isinstance(workload_command, list): + raise SGLangMeasurementError("measurement.workload_command must be a list of command arguments") + workload = subprocess.run(workload_command, check=False, capture_output=True, text=True) + events.append( + { + "type": "workload", + "returncode": workload.returncode, + "stdout": workload.stdout[-2000:], + "stderr": workload.stderr[-2000:], + } + ) + else: + time.sleep(duration) + events.append({"type": "idle_collection", "duration_seconds": duration}) + finally: + sampler_error = None + if sampler is not None: + try: + sampler.stop() + except SGLangMeasurementError as exc: + sampler_error = exc + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + if sampler_error is not None: + raise sampler_error + + elapsed, avg_power, energy = _power_summary(events, duration) + row = { + "experiment": (config.get("experiment", {}) or {}).get("name", "aether-sglang"), + "backend": "sglang", + "scenario_id": "sglang-real", + "model": (config.get("model", {}) or {}).get("name", "unknown-model"), + "hardware": "real-gpu", + "routing_policy": "real", + "routing_pool": "sglang", + "scheduling_policy": "real", + "scheduling_action": "observed", + "elapsed_seconds": round(elapsed, 6), + "energy_j": round(energy, 6), + "avg_power_w": round(avg_power, 6), + } + write_csv([row], str(output_dir / "summary.csv")) + write_jsonl(events, str(output_dir / "events.jsonl")) + with (output_dir / "command.json").open("w", encoding="utf-8") as handle: + json.dump({"command": command}, handle, indent=2) + return row, events diff --git a/src/aether/results.py b/src/aether/results.py new file mode 100644 index 0000000..7e0dbdf --- /dev/null +++ b/src/aether/results.py @@ -0,0 +1,76 @@ +"""Normalized result writers.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any, Dict, Iterable, List + + +SUMMARY_FIELDS = [ + "experiment", + "backend", + "scenario_id", + "model", + "hardware", + "routing_policy", + "routing_pool", + "scheduling_policy", + "scheduling_action", + "context_tokens", + "prompt_tokens", + "generated_tokens", + "target_concurrency", + "active_sequences", + "kv_compression_ratio", + "kv_bytes_per_token", + "effective_kv_bytes_per_token", + "max_inflight_sequences", + "prefill_seconds", + "decode_seconds", + "elapsed_seconds", + "energy_j", + "avg_power_w", + "tokens_per_second", + "tokens_per_watt", + "tokens_per_joule", + "quality_score", + "quality_normalized_ipw", + "swap_bytes", + "recompute_tokens", + "ttft_ms", + "tbt_ms", + "slo_violations", +] + + +def ensure_parent(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + + +def normalized_row(row: Dict[str, Any]) -> Dict[str, Any]: + return {field: row.get(field, "") for field in SUMMARY_FIELDS} + + +def write_csv(rows: Iterable[Dict[str, Any]], path: str) -> None: + output_path = Path(path) + ensure_parent(output_path) + normalized_rows = [normalized_row(row) for row in rows] + with output_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=SUMMARY_FIELDS) + writer.writeheader() + writer.writerows(normalized_rows) + + +def write_jsonl(events: Iterable[Dict[str, Any]], path: str) -> None: + output_path = Path(path) + ensure_parent(output_path) + with output_path.open("w", encoding="utf-8") as handle: + for event in events: + handle.write(json.dumps(event, sort_keys=True) + "\n") + + +def read_csv_rows(path: str) -> List[Dict[str, str]]: + with Path(path).open("r", encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) diff --git a/src/aether/simulator.py b/src/aether/simulator.py new file mode 100644 index 0000000..3cac833 --- /dev/null +++ b/src/aether/simulator.py @@ -0,0 +1,264 @@ +"""Scenario expansion and GreenServe/IPW simulation.""" + +from __future__ import annotations + +import itertools +from typing import Any, Dict, Iterable, List, Mapping + +from . import formulas +from .config import as_list, experiment_name + + +def _first(items: Iterable[Dict[str, Any]], default: Dict[str, Any]) -> Dict[str, Any]: + for item in items: + return dict(item) + return dict(default) + + +def _hardware_profiles(config: Mapping[str, Any]) -> List[Dict[str, Any]]: + simulation = dict(config.get("simulation", {}) or {}) + profiles = simulation.get("hardware_profiles") or [] + if not profiles: + profiles = [{"name": "default-gpu", "tdp_w": 700, "vram_gb": 80, "gpu_count": 1}] + return [dict(profile) for profile in profiles] + + +def _model_profile(config: Mapping[str, Any]) -> Dict[str, Any]: + model = dict(config.get("model", {}) or {}) + model.setdefault("name", "unknown-model") + model.setdefault("layers", 32) + model.setdefault("kv_heads", 8) + model.setdefault("head_dim", 128) + model.setdefault("kv_dtype_bytes", 2) + model.setdefault("weight_gb", 0) + model.setdefault("quality_score", 1.0) + return model + + +def _workload(config: Mapping[str, Any]) -> Dict[str, Any]: + workload = dict(config.get("workload", {}) or {}) + workload.setdefault("prompt_tokens", 4096) + workload.setdefault("generated_tokens", 256) + workload.setdefault("target_concurrency", 1) + workload.setdefault("preemption_probability", 0.0) + workload.setdefault("ttft_slo_ms", 2000) + workload.setdefault("tbt_slo_ms", 200) + return workload + + +def _sweeps(config: Mapping[str, Any], workload: Mapping[str, Any]) -> Dict[str, List[Any]]: + simulation = dict(config.get("simulation", {}) or {}) + sweep = dict(simulation.get("sweeps", simulation.get("sweep", {})) or {}) + return { + "context_lengths": as_list(sweep.get("context_lengths"), [workload["prompt_tokens"]]), + "kv_compression_ratios": as_list(sweep.get("kv_compression_ratios"), [1.0]), + "routing_policies": as_list(sweep.get("routing_policies"), ["single_pool"]), + "scheduling_policies": as_list(sweep.get("scheduling_policies"), ["none"]), + } + + +def _select_pool(config: Mapping[str, Any], policy: str, context_tokens: int, hardware: Mapping[str, Any]) -> Dict[str, Any]: + simulation = dict(config.get("simulation", {}) or {}) + pools = [dict(pool) for pool in simulation.get("routing_pools", []) or []] + if not pools: + return { + "name": "default", + "gpu_count": int(hardware.get("gpu_count", 1)), + "max_context_tokens": context_tokens, + } + if policy == "fleetopt": + candidates = [pool for pool in pools if int(pool.get("max_context_tokens", context_tokens)) >= context_tokens] + if candidates: + return sorted(candidates, key=lambda item: int(item.get("max_context_tokens", context_tokens)))[0] + return pools[0] + + +def _scheduler_overhead( + policy: str, + context_tokens: int, + active_sequences: int, + effective_kv_bytes_per_token: float, + workload: Mapping[str, Any], + hardware: Mapping[str, Any], + prefill_tps: float, +) -> Dict[str, Any]: + probability = float(workload.get("preemption_probability", 0.0)) + preempted_sequences = int(round(active_sequences * probability)) + if policy == "none" or preempted_sequences <= 0: + return {"action": "none", "seconds": 0.0, "energy_j": 0.0, "swap_bytes": 0.0, "recompute_tokens": 0} + + gpu_count = int(hardware.get("gpu_count", 1)) + tdp_w = float(hardware.get("tdp_w", 700)) + pcie_gbps = float(hardware.get("pcie_bandwidth_gbps", 64)) + risk_weight = float(hardware.get("scheduler_risk_weight", 1.0)) + preempted_context_tokens = context_tokens * preempted_sequences + kv_bytes = effective_kv_bytes_per_token * preempted_context_tokens + + recompute = formulas.recompute_cost(preempted_context_tokens, prefill_tps, tdp_w, gpu_count) + swap = formulas.swap_cost(kv_bytes, pcie_gbps, tdp_w, gpu_count) + + if policy == "recompute": + action = "recompute" + elif policy == "swap": + action = "swap" + else: + recompute_score = formulas.risk_adjusted_cost(recompute["energy_j"], recompute["uncertainty_j"], risk_weight) + swap_score = formulas.risk_adjusted_cost(swap["energy_j"], swap["uncertainty_j"], risk_weight) + action = "recompute" if recompute_score < swap_score else "swap" + + selected = recompute if action == "recompute" else swap + return { + "action": action, + "seconds": selected["seconds"], + "energy_j": selected["energy_j"], + "swap_bytes": kv_bytes if action == "swap" else 0.0, + "recompute_tokens": preempted_context_tokens if action == "recompute" else 0, + } + + +def simulate(config: Mapping[str, Any]) -> List[Dict[str, Any]]: + model = _model_profile(config) + workload = _workload(config) + sweeps = _sweeps(config, workload) + rows = [] + exp_name = experiment_name(config) + for hardware, context_tokens, compression_ratio, routing_policy, scheduling_policy in itertools.product( + _hardware_profiles(config), + sweeps["context_lengths"], + sweeps["kv_compression_ratios"], + sweeps["routing_policies"], + sweeps["scheduling_policies"], + ): + rows.append( + simulate_one( + config, + exp_name, + model, + workload, + hardware, + int(context_tokens), + float(compression_ratio), + str(routing_policy), + str(scheduling_policy), + ) + ) + return rows + + +def simulate_one( + config: Mapping[str, Any], + exp_name: str, + model: Mapping[str, Any], + workload: Mapping[str, Any], + hardware: Mapping[str, Any], + context_tokens: int, + compression_ratio: float, + routing_policy: str, + scheduling_policy: str, +) -> Dict[str, Any]: + pool = _select_pool(config, routing_policy, context_tokens, hardware) + gpu_count = int(pool.get("gpu_count", hardware.get("gpu_count", 1))) + tdp_w = float(hardware.get("tdp_w", 700)) + prefill_tps = float(hardware.get("prefill_tps_per_gpu", 5000)) * gpu_count + decode_tps = float(hardware.get("decode_tps_per_gpu", 250)) * gpu_count + target_concurrency = int(workload.get("target_concurrency", 1)) + generated_tokens_per_request = int(workload.get("generated_tokens", 256)) + prompt_tokens = int(workload.get("prompt_tokens", context_tokens)) + + kv_bytes = formulas.kv_bytes_per_token( + int(model["layers"]), + int(model["kv_heads"]), + int(model["head_dim"]), + int(model["kv_dtype_bytes"]), + ) + effective_kv = formulas.effective_kv_bytes( + kv_bytes, + compression_ratio, + float(model.get("kv_metadata_bytes_per_token", 0.0)), + ) + model_weight_gb_per_gpu = float( + model.get("weight_gb_per_gpu", formulas.safe_div(float(model.get("weight_gb", 0.0)), max(1, gpu_count))) + ) + available_vram = formulas.available_kv_vram_bytes( + float(hardware.get("vram_gb", 80)), + model_weight_gb_per_gpu, + float(hardware.get("gpu_memory_utilization", 0.9)), + ) * gpu_count + max_inflight = formulas.max_inflight_sequences(available_vram, effective_kv, context_tokens) + active_sequences = max(0, min(target_concurrency, max_inflight)) + useful_concurrency_fraction = min(1.0, formulas.safe_div(max_inflight, max(1, target_concurrency))) + effective_decode_tps = max(1.0, decode_tps * useful_concurrency_fraction) + + total_generated_tokens = active_sequences * generated_tokens_per_request + prefill_seconds = formulas.safe_div(active_sequences * prompt_tokens, max(1.0, prefill_tps)) + decode_seconds = formulas.safe_div(total_generated_tokens, effective_decode_tps) + prefill_power = gpu_count * tdp_w * float(hardware.get("prefill_power_fraction", 0.86)) + decode_power = gpu_count * tdp_w * float(hardware.get("decode_power_fraction", 0.43)) + prefill_energy = formulas.phase_energy_j(prefill_seconds, prefill_power) + decode_energy = formulas.phase_energy_j(decode_seconds, decode_power) + + overhead = _scheduler_overhead( + scheduling_policy, + context_tokens, + active_sequences, + effective_kv, + workload, + dict(hardware, gpu_count=gpu_count), + prefill_tps, + ) + elapsed = prefill_seconds + decode_seconds + overhead["seconds"] + energy = prefill_energy + decode_energy + overhead["energy_j"] + avg_power = formulas.safe_div(energy, elapsed) + tokens_per_second = formulas.safe_div(total_generated_tokens, elapsed) + tokens_per_watt = formulas.safe_div(tokens_per_second, avg_power) + tokens_per_joule = formulas.safe_div(total_generated_tokens, energy) + ttft_ms = 1000.0 * prefill_seconds + tbt_ms = 1000.0 * formulas.safe_div(decode_seconds, max(1, generated_tokens_per_request)) + slo_violations = int(ttft_ms > float(workload.get("ttft_slo_ms", 2000))) + int( + tbt_ms > float(workload.get("tbt_slo_ms", 200)) + ) + + scenario_id = "%s-%s-ctx%s-kv%s-%s-%s" % ( + exp_name, + hardware.get("name", "hardware"), + context_tokens, + compression_ratio, + routing_policy, + scheduling_policy, + ) + quality = float(model.get("quality_score", 1.0)) + return { + "experiment": exp_name, + "backend": "simulation", + "scenario_id": scenario_id, + "model": model.get("name", "unknown-model"), + "hardware": hardware.get("name", "unknown-hardware"), + "routing_policy": routing_policy, + "routing_pool": pool.get("name", "default"), + "scheduling_policy": scheduling_policy, + "scheduling_action": overhead["action"], + "context_tokens": context_tokens, + "prompt_tokens": prompt_tokens, + "generated_tokens": total_generated_tokens, + "target_concurrency": target_concurrency, + "active_sequences": active_sequences, + "kv_compression_ratio": compression_ratio, + "kv_bytes_per_token": kv_bytes, + "effective_kv_bytes_per_token": round(effective_kv, 6), + "max_inflight_sequences": max_inflight, + "prefill_seconds": round(prefill_seconds, 6), + "decode_seconds": round(decode_seconds, 6), + "elapsed_seconds": round(elapsed, 6), + "energy_j": round(energy, 6), + "avg_power_w": round(avg_power, 6), + "tokens_per_second": round(tokens_per_second, 6), + "tokens_per_watt": round(tokens_per_watt, 9), + "tokens_per_joule": round(tokens_per_joule, 9), + "quality_score": quality, + "quality_normalized_ipw": round(quality * tokens_per_joule, 9), + "swap_bytes": round(overhead["swap_bytes"], 6), + "recompute_tokens": overhead["recompute_tokens"], + "ttft_ms": round(ttft_ms, 6), + "tbt_ms": round(tbt_ms, 6), + "slo_violations": slo_violations, + } diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..8d1f7a1 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,48 @@ +from aether.cli import main +from aether.results import read_csv_rows + + +def test_cli_simulate(tmp_path): + output = tmp_path / "sim.csv" + rc = main(["simulate", "--config", "experiments/greensserve/baseline.yaml", "--out", str(output)]) + + assert rc == 0 + rows = read_csv_rows(str(output)) + assert len(rows) == 96 + + +def test_cli_mock_launch(tmp_path): + output = tmp_path / "mock" + rc = main( + [ + "launch", + "--backend", + "mock", + "--config", + "experiments/greensserve/mock_measurement.yaml", + "--out", + str(output), + ] + ) + + assert rc == 0 + rows = read_csv_rows(str(output / "summary.csv")) + assert rows[0]["backend"] == "mock" + assert (output / "events.jsonl").exists() + + +def test_cli_sglang_backend_is_optional(tmp_path): + output = tmp_path / "sglang" + rc = main( + [ + "launch", + "--backend", + "sglang", + "--config", + "experiments/greensserve/sglang_real.yaml", + "--out", + str(output), + ] + ) + + assert rc == 2 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..d94c0d3 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,22 @@ +from aether.config import render_sglang_args + + +def test_render_sglang_args_sorts_and_renders_flags(): + config = { + "sglang": { + "args": { + "port": 30000, + "enable-metrics": True, + "disabled": False, + "none": None, + }, + "extra_args": ["--trust-remote-code"], + } + } + + assert render_sglang_args(config) == [ + "--enable-metrics", + "--port", + "30000", + "--trust-remote-code", + ] diff --git a/tests/test_formulas.py b/tests/test_formulas.py new file mode 100644 index 0000000..7fafbac --- /dev/null +++ b/tests/test_formulas.py @@ -0,0 +1,18 @@ +from aether import formulas + + +def test_kv_bytes_per_token(): + assert formulas.kv_bytes_per_token(layers=80, kv_heads=8, head_dim=128, kv_dtype_bytes=2) == 327680 + + +def test_compression_improves_capacity(): + available = formulas.available_kv_vram_bytes(80, 40, 0.9) + kv = formulas.kv_bytes_per_token(80, 8, 128, 2) + base = formulas.max_inflight_sequences(available, kv, 4096) + compressed = formulas.max_inflight_sequences(available, formulas.effective_kv_bytes(kv, 0.5), 4096) + + assert compressed == base * 2 or compressed == base * 2 + 1 + + +def test_risk_adjusted_cost(): + assert formulas.risk_adjusted_cost(10, 2, 1.5) == 13 diff --git a/tests/test_results.py b/tests/test_results.py new file mode 100644 index 0000000..d0220bf --- /dev/null +++ b/tests/test_results.py @@ -0,0 +1,11 @@ +from aether.results import SUMMARY_FIELDS, read_csv_rows, write_csv + + +def test_write_csv_stable_schema(tmp_path): + path = tmp_path / "out.csv" + write_csv([{"experiment": "x", "backend": "mock", "extra": "ignored"}], str(path)) + rows = read_csv_rows(str(path)) + + assert list(rows[0].keys()) == SUMMARY_FIELDS + assert rows[0]["experiment"] == "x" + assert rows[0]["backend"] == "mock" diff --git a/tests/test_simulator.py b/tests/test_simulator.py new file mode 100644 index 0000000..f4296e1 --- /dev/null +++ b/tests/test_simulator.py @@ -0,0 +1,21 @@ +from aether.config import load_config +from aether.simulator import simulate + + +def test_baseline_sweep_expands(): + config = load_config("experiments/greensserve/baseline.yaml") + rows = simulate(config) + + assert len(rows) == 4 * 3 * 2 * 4 + assert rows[0]["backend"] == "simulation" + assert "tokens_per_joule" in rows[0] + assert max(row["max_inflight_sequences"] for row in rows) > 0 + assert max(row["generated_tokens"] for row in rows) > 0 + + +def test_bayesian_scheduler_selects_action(): + config = load_config("experiments/greensserve/mock_measurement.yaml") + rows = simulate(config) + + assert len(rows) == 1 + assert rows[0]["scheduling_action"] in {"recompute", "swap", "none"} diff --git a/third_party/sglang b/third_party/sglang new file mode 160000 index 0000000..127b9e3 --- /dev/null +++ b/third_party/sglang @@ -0,0 +1 @@ +Subproject commit 127b9e3283f7c2a43234b852ff5c9f1796d53624 diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..b450b5e --- /dev/null +++ b/uv.lock @@ -0,0 +1,153 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[[package]] +name = "aether" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, +] + +[package.optional-dependencies] +sglang = [ + { name = "pynvml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] +sglang = [ + { name = "pynvml" }, +] + +[package.metadata] +requires-dist = [ + { name = "pynvml", marker = "extra == 'sglang'", specifier = ">=11.5" }, + { name = "pyyaml", specifier = ">=5.4" }, +] +provides-extras = ["sglang"] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.3" }] +sglang = [{ name = "pynvml", specifier = ">=11.5" }] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "nvidia-ml-py" +version = "13.595.45" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/49/c29f6e30d8662d2e94fef17739ea7309cc76aba269922ae999e4cc07f268/nvidia_ml_py-13.595.45.tar.gz", hash = "sha256:c9f34897fe0441ff35bc8f35baf80f830a20b0f4e6ce71e0a325bc0e66acf079", size = 50780, upload-time = "2026-03-19T16:59:44.956Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/24/fc256107d23597fa33d319505ce77160fa1a2349c096d01901ffc7cb7fc4/nvidia_ml_py-13.595.45-py3-none-any.whl", hash = "sha256:b65a7977f503d56154b14d683710125ef93594adb63fbf7e559336e3318f1376", size = 51776, upload-time = "2026-03-19T16:59:43.603Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pynvml" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-ml-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/57/da7dc63a79f59e082e26a66ac02d87d69ea316b35b35b7a00d82f3ce3d2f/pynvml-13.0.1.tar.gz", hash = "sha256:1245991d9db786b4d2f277ce66869bd58f38ac654e38c9397d18f243c8f6e48f", size = 35226, upload-time = "2025-09-05T20:33:25.377Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/4a/cac76c174bb439a0c46c9a4413fcbea5c6cabfb01879f7bbdb9fdfaed76c/pynvml-13.0.1-py3-none-any.whl", hash = "sha256:e2b20e0a501eeec951e2455b7ab444759cf048e0e13a57b08049fa2775266aa8", size = 28810, upload-time = "2025-09-05T20:33:24.13Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] From ca1d92c4784b7d27cf168ffe3253a3d1753d852a Mon Sep 17 00:00:00 2001 From: Zhisheng Ye Date: Mon, 25 May 2026 13:00:58 +0800 Subject: [PATCH 2/4] Run CI on feature branch pushes --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffac0ab..a1c9812 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,6 @@ name: CI on: push: - branches: [master, main] pull_request: jobs: From 348a7fd6d6d429ec98b3e61ada1f9522feaaa88d Mon Sep 17 00:00:00 2001 From: Zhisheng Ye Date: Mon, 25 May 2026 13:02:03 +0800 Subject: [PATCH 3/4] Opt CI into Node 24 actions runtime --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1c9812..c0cef44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,8 @@ name: CI +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + on: push: pull_request: From fb3beb353d252bdc0fd76221c05d0f91bc992a96 Mon Sep 17 00:00:00 2001 From: Zhisheng Ye Date: Mon, 25 May 2026 15:36:22 +0800 Subject: [PATCH 4/4] Exercise SGLang CPU path in CI --- .github/uv-sglang-cpu.toml | 15 ++ .github/workflows/ci.yml | 71 +++++++ AGENTS.md | 13 +- README.md | 23 ++- docs/real-data-collection.md | 46 ++++- experiments/greensserve/sglang_cpu_ci.yaml | 62 ++++++ src/aether/cli.py | 2 +- src/aether/measurement/sglang.py | 209 ++++++++++++++++++++- tests/test_sglang_measurement.py | 88 +++++++++ 9 files changed, 507 insertions(+), 22 deletions(-) create mode 100644 .github/uv-sglang-cpu.toml create mode 100644 experiments/greensserve/sglang_cpu_ci.yaml create mode 100644 tests/test_sglang_measurement.py diff --git a/.github/uv-sglang-cpu.toml b/.github/uv-sglang-cpu.toml new file mode 100644 index 0000000..f73d398 --- /dev/null +++ b/.github/uv-sglang-cpu.toml @@ -0,0 +1,15 @@ +[[index]] +name = "torch" +url = "https://download.pytorch.org/whl/cpu" + +[[index]] +name = "torchvision" +url = "https://download.pytorch.org/whl/cpu" + +[[index]] +name = "torchaudio" +url = "https://download.pytorch.org/whl/cpu" + +[[index]] +name = "triton" +url = "https://download.pytorch.org/whl/cpu" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0cef44..fa29da5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,3 +47,74 @@ jobs: - name: Verify SGLang patch applies run: | git -C third_party/sglang apply --check ../../patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch + + sglang-cpu: + name: SGLang CPU launch + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + UV_CONFIG_FILE: ${{ github.workspace }}/.github/uv-sglang-cpu.toml + SGLANG_USE_CPU_ENGINE: "1" + steps: + - name: Check out repo + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Set up Python + run: uv python install 3.13 + + - name: Install system packages for SGLang CPU + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y google-perftools libtbb-dev libnuma-dev numactl + + - name: Install Aether dependencies + run: uv sync --group dev --group sglang + + - name: Apply Aether SGLang metrics patch + run: | + git -C third_party/sglang apply ../../patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch + test -f third_party/sglang/AETHER_IPW_METRICS.md + + - name: Install SGLang CPU from pinned submodule + run: | + cp third_party/sglang/python/pyproject_cpu.toml third_party/sglang/python/pyproject.toml + cp third_party/sglang/sgl-kernel/pyproject_cpu.toml third_party/sglang/sgl-kernel/pyproject.toml + uv pip install --upgrade pip setuptools wheel + uv pip install third_party/sglang/python + uv pip install third_party/sglang/sgl-kernel + uv run python -c "import sglang, sgl_kernel; print('sglang import ok', sglang.__file__); print('sgl_kernel import ok', sgl_kernel.__file__)" + + - name: Run Aether against real SGLang CPU server + env: + LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu + LD_PRELOAD: ${{ github.workspace }}/.venv/lib/libiomp5.so:/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4:/usr/lib/x86_64-linux-gnu/libtbbmalloc.so.2 + run: | + uv run aether doctor --config experiments/greensserve/sglang_cpu_ci.yaml + timeout 15m uv run aether launch --backend sglang --config experiments/greensserve/sglang_cpu_ci.yaml --out /tmp/aether-sglang-cpu + + - name: Show Aether SGLang CPU outputs + if: always() + run: | + echo "===== command.json =====" + test -f /tmp/aether-sglang-cpu/command.json && cat /tmp/aether-sglang-cpu/command.json || true + echo "===== summary.csv =====" + test -f /tmp/aether-sglang-cpu/summary.csv && cat /tmp/aether-sglang-cpu/summary.csv || true + echo "===== events.jsonl =====" + test -f /tmp/aether-sglang-cpu/events.jsonl && head -100 /tmp/aether-sglang-cpu/events.jsonl || true + echo "===== sglang.log tail =====" + test -f /tmp/aether-sglang-cpu/sglang.log && tail -200 /tmp/aether-sglang-cpu/sglang.log || true + + - name: Upload SGLang CPU artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: aether-sglang-cpu + path: /tmp/aether-sglang-cpu + if-no-files-found: ignore diff --git a/AGENTS.md b/AGENTS.md index 9ee5469..9187b30 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,6 +19,12 @@ uv run aether launch --backend mock --config experiments/greensserve/mock_measur git -C third_party/sglang apply --check ../../patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch ``` +GitHub CI also runs a real SGLang CPU integration job. It installs SGLang from +the pinned submodule using SGLang's `pyproject_cpu.toml`, applies the Aether +metrics patch, launches `python -m sglang.launch_server --device cpu`, sends a +tiny `/generate` request through `aether launch --backend sglang`, and prints +`summary.csv`, `events.jsonl`, and `sglang.log`. + ## Network Proxy Preference When a download or dependency fetch appears stuck or blocked by network @@ -36,7 +42,8 @@ ALL_PROXY=socks5://127.0.0.1:10808 ## Repo Rules - Keep real SGLang/GPU measurement code optional. Importing `aether` and running - tests must not require CUDA, NVML, SGLang, or a GPU. + unit tests must not require CUDA, NVML, SGLang, or a GPU. CI may install + SGLang separately for the dedicated CPU integration job. - Use mock measurement as the correctness gate for launcher, collector, and result-writer behavior. - Keep normalized CSV/JSONL schemas shared across simulation, mock, and real @@ -48,5 +55,5 @@ ALL_PROXY=socks5://127.0.0.1:10808 recompute/swap, or DVFS behavior changes without a new design doc. - SGLang `v0.5.12` does not fully install on local macOS arm64 in this workspace because `sgl-deep-gemm==0.1.0` provides Linux wheels only. Use - macOS for patch-apply and Aether mock validation; use Linux GPU hosts for - real SGLang serving tests. + macOS for patch-apply and Aether mock validation; use GitHub Actions or Linux + hosts for SGLang CPU serving tests, and Linux GPU hosts for NVML/GPU runs. diff --git a/README.md b/README.md index a3009f7..b6fe27b 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,19 @@ supports pure simulation and a deterministic mock measurement backend that validates the same result schema used by future SGLang/GPU runs. The GitHub CI is CPU-only. It uses uv with Python 3.13, runs tests and smoke -commands, and verifies the SGLang metrics patch applies to the pinned submodule. +commands, verifies the SGLang metrics patch applies to the pinned submodule, +and has a dedicated job that installs SGLang's CPU source build, launches an +actual SGLang CPU server, sends an Aether-managed request, and prints the +normalized outputs. -Real GPU experiments are kept separate. The SGLang backend is present, but it -only runs when SGLang, CUDA/NVML, and a workload are available in the active -environment. +Real GPU experiments are kept separate. The SGLang backend is present, but GPU +power sampling only runs when SGLang, CUDA/NVML, and a workload are available +in the active environment. -Note: SGLang `v0.5.12` real serving is Linux/GPU-oriented. On this local macOS -arm64 machine, full install is blocked by upstream Linux-only `sgl-deep-gemm` -wheels; see `docs/real-data-collection.md` for the smoke-test result. +Note: SGLang `v0.5.12` real serving is Linux-oriented. On this local macOS arm64 +machine, full install is blocked by upstream Linux-only `sgl-deep-gemm` wheels; +see `docs/real-data-collection.md` for the smoke-test result and the Linux CPU +CI path. ## Install @@ -62,6 +66,7 @@ Examples live in `experiments/greensserve/`. - `baseline.yaml`: pure simulation sweep. - `mock_measurement.yaml`: CPU-only launcher and collector validation. +- `sglang_cpu_ci.yaml`: real SGLang CPU launch used by GitHub Actions. - `sglang_real.yaml`: template for future SGLang/GPU measurement. The SGLang config accepts structured args: @@ -91,4 +96,6 @@ Real collection is documented in `docs/real-data-collection.md`. In short: 4. Run `uv run aether launch --backend sglang --config experiments/greensserve/sglang_real.yaml --out results/real-run`. 5. Use `summary.csv`, `events.jsonl`, SGLang logs, and NVML samples for analysis. -The current test suite does not require SGLang, CUDA, NVML, or a GPU. +The unit test suite does not require SGLang, CUDA, NVML, or a GPU. GitHub CI +adds an integration job that installs SGLang CPU separately and exercises the +same Aether result writer against a live local SGLang server. diff --git a/docs/real-data-collection.md b/docs/real-data-collection.md index 55789c2..5951931 100644 --- a/docs/real-data-collection.md +++ b/docs/real-data-collection.md @@ -1,7 +1,47 @@ # Real SGLang Data Collection -Real measurement is separate from CPU validation. Use it only on a machine with -NVIDIA GPUs, working drivers, CUDA, NVML, and an installed SGLang environment. +Real measurement is separate from mock validation. Aether can run SGLang in CPU +mode for integration checks, and it can run GPU/NVML collection on a machine +with NVIDIA GPUs, working drivers, CUDA, NVML, and an installed SGLang +environment. + +## CI CPU SGLang Check + +GitHub Actions has a dedicated `SGLang CPU launch` job that performs a real +source install from the pinned `third_party/sglang` submodule: + +1. configure uv to use PyTorch CPU wheels +2. install the Linux system libraries required by SGLang's CPU backend +3. apply `patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch` +4. copy SGLang's `pyproject_cpu.toml` files into place +5. install `third_party/sglang/python` and `third_party/sglang/sgl-kernel` +6. run `aether launch --backend sglang` with + `experiments/greensserve/sglang_cpu_ci.yaml` + +The CI log prints `command.json`, `summary.csv`, `events.jsonl`, and the tail +of `sglang.log`. The run is intentionally tiny: it uses +`hf-internal-testing/tiny-random-LlamaForCausalLM`, `--load-format dummy`, +`--device cpu`, and a two-token `/generate` request. This validates Aether's +real launcher, health polling, request path, metric extraction, and normalized +result writer without requiring a GPU. + +The same CPU path can be run manually on Linux: + +```bash +export UV_CONFIG_FILE=$PWD/.github/uv-sglang-cpu.toml +export SGLANG_USE_CPU_ENGINE=1 +sudo apt-get update +sudo apt-get install --no-install-recommends -y google-perftools libtbb-dev libnuma-dev numactl +uv sync --group dev --group sglang +git -C third_party/sglang apply ../../patches/sglang/v0.5.12/0001-aether-ipw-metrics-scaffold.patch +cp third_party/sglang/python/pyproject_cpu.toml third_party/sglang/python/pyproject.toml +cp third_party/sglang/sgl-kernel/pyproject_cpu.toml third_party/sglang/sgl-kernel/pyproject.toml +uv pip install third_party/sglang/python +uv pip install third_party/sglang/sgl-kernel +export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu +export LD_PRELOAD=$PWD/.venv/lib/libiomp5.so:/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4:/usr/lib/x86_64-linux-gnu/libtbbmalloc.so.2 +uv run aether launch --backend sglang --config experiments/greensserve/sglang_cpu_ci.yaml --out results/sglang-cpu-ci +``` ## Prepare SGLang @@ -56,7 +96,7 @@ The SGLang backend will: 2. launch `python -m sglang.launch_server` 3. wait for `/health` 4. sample NVML power when `measurement.nvml_enabled` is true -5. optionally run the configured workload command +5. run configured HTTP requests or an optional workload command 6. write normalized `summary.csv` and `events.jsonl` ## Data To Record diff --git a/experiments/greensserve/sglang_cpu_ci.yaml b/experiments/greensserve/sglang_cpu_ci.yaml new file mode 100644 index 0000000..93c6778 --- /dev/null +++ b/experiments/greensserve/sglang_cpu_ci.yaml @@ -0,0 +1,62 @@ +experiment: + name: greensserve-sglang-cpu-ci + tags: [sglang, cpu, ci, real-launch] + +model: + name: tiny-random-llama-sglang-cpu + layers: 2 + kv_heads: 4 + head_dim: 16 + kv_dtype_bytes: 2 + weight_gb: 0.01 + quality_score: 1.0 + +sglang: + args: + model-path: hf-internal-testing/tiny-random-LlamaForCausalLM + host: 127.0.0.1 + port: 30080 + device: cpu + context-length: 256 + max-total-tokens: 512 + load-format: dummy + tensor-parallel-size: 1 + disable-overlap-schedule: true + extra_args: [] + +measurement: + duration_seconds: 1 + sample_hz: 1 + nvml_enabled: false + health_timeout_seconds: 240 + request_timeout_seconds: 120 + requests: + - endpoint: /generate + payload: + text: "Human: Say hello in one short sentence.\n\nAssistant:" + sampling_params: + temperature: 0.0 + max_new_tokens: 2 + stream: false + +workload: + prompt_tokens: 16 + generated_tokens: 2 + target_concurrency: 1 + preemption_probability: 0.0 + +simulation: + hardware_profiles: + - name: github-actions-cpu + gpu_count: 1 + tdp_w: 65 + vram_gb: 8 + gpu_memory_utilization: 0.5 + pcie_bandwidth_gbps: 16 + prefill_tps_per_gpu: 50 + decode_tps_per_gpu: 5 + sweeps: + context_lengths: [256] + kv_compression_ratios: [1.0] + routing_policies: [single_pool] + scheduling_policies: [none] diff --git a/src/aether/cli.py b/src/aether/cli.py index e144ade..f26bf01 100644 --- a/src/aether/cli.py +++ b/src/aether/cli.py @@ -62,7 +62,7 @@ def _cmd_doctor(args: argparse.Namespace) -> int: print("simulation_scenarios: %s" % len(rows)) print("sglang_args: %s" % " ".join(render_sglang_args(config))) print("cpu_safe: true") - print("real_sglang_requires: SGLang, CUDA, NVML, GPU, and a workload") + print("real_sglang_requires: SGLang plus a workload; GPU/NVML only when nvml_enabled=true") return 0 diff --git a/src/aether/measurement/sglang.py b/src/aether/measurement/sglang.py index 9c14327..6401835 100644 --- a/src/aether/measurement/sglang.py +++ b/src/aether/measurement/sglang.py @@ -14,9 +14,10 @@ import time from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, Tuple -from urllib.error import URLError -from urllib.request import urlopen +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen +from .. import formulas from ..config import render_sglang_args from ..results import write_csv, write_jsonl @@ -33,11 +34,17 @@ def _require_sglang() -> None: ) -def _health_url(config: Mapping[str, Any]) -> str: +def _server_base_url(config: Mapping[str, Any]) -> str: args = dict((config.get("sglang", {}) or {}).get("args", {}) or {}) host = str(args.get("host", "127.0.0.1")) + if host in {"0.0.0.0", "::"}: + host = "127.0.0.1" port = str(args.get("port", "30000")) - return "http://%s:%s/health" % (host, port) + return "http://%s:%s" % (host, port) + + +def _health_url(config: Mapping[str, Any]) -> str: + return _server_base_url(config) + "/health" def _wait_for_health(url: str, timeout_s: float) -> bool: @@ -119,6 +126,172 @@ def _power_summary(events: List[Dict[str, Any]], fallback_duration: float) -> Tu return elapsed, avg_power, avg_power * elapsed +def _post_json(url: str, payload: Mapping[str, Any], timeout_s: float) -> Tuple[int, Dict[str, Any]]: + body = json.dumps(payload).encode("utf-8") + request = Request( + url, + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=timeout_s) as response: + raw = response.read().decode("utf-8") + parsed = json.loads(raw) if raw else {} + if not isinstance(parsed, dict): + parsed = {"response": parsed} + return int(response.status), parsed + except HTTPError as exc: + raw = exc.read().decode("utf-8", errors="replace") + raise SGLangMeasurementError("SGLang request failed with HTTP %s: %s" % (exc.code, raw[-2000:])) from exc + except (OSError, URLError, json.JSONDecodeError) as exc: + raise SGLangMeasurementError("SGLang request failed: %s" % exc) from exc + + +def _as_int(value: Any) -> int: + if value is None: + return 0 + try: + return int(value) + except (TypeError, ValueError): + return 0 + + +def _as_float(value: Any) -> float: + if value is None: + return 0.0 + try: + return float(value) + except (TypeError, ValueError): + return 0.0 + + +def _extract_response_metrics(response: Mapping[str, Any]) -> Dict[str, Any]: + """Extract comparable token and latency metrics from SGLang or OpenAI output.""" + + usage = response.get("usage") + if isinstance(usage, dict): + prompt_tokens = _as_int(usage.get("prompt_tokens")) + completion_tokens = _as_int(usage.get("completion_tokens")) + total_tokens = _as_int(usage.get("total_tokens")) or prompt_tokens + completion_tokens + return { + "prompt_tokens": prompt_tokens, + "generated_tokens": completion_tokens, + "total_tokens": total_tokens, + } + + responses: List[Mapping[str, Any]] + if isinstance(response.get("response"), list): + responses = [item for item in response["response"] if isinstance(item, dict)] + else: + responses = [response] + + prompt_tokens = 0 + generated_tokens = 0 + total_tokens = 0 + latencies = [] + ttfts = [] + tbts = [] + for item in responses: + meta = item.get("meta_info", {}) + if not isinstance(meta, dict): + meta = {} + prompt_tokens += _as_int(meta.get("prompt_tokens") or meta.get("input_tokens")) + generated_tokens += _as_int(meta.get("completion_tokens") or meta.get("output_tokens")) + total_tokens += _as_int(meta.get("total_tokens")) + latency = _as_float(meta.get("e2e_latency") or meta.get("latency")) + if latency > 0: + latencies.append(latency) + ttft = _as_float( + meta.get("ttft") + or meta.get("time_to_first_token") + or meta.get("first_token_latency") + or meta.get("first_token_latency_s") + ) + if ttft > 0: + ttfts.append(ttft) + tbt = _as_float(meta.get("tbt") or meta.get("inter_token_latency") or meta.get("decode_token_latency")) + if tbt > 0: + tbts.append(tbt) + + if total_tokens == 0: + total_tokens = prompt_tokens + generated_tokens + + metrics: Dict[str, Any] = { + "prompt_tokens": prompt_tokens, + "generated_tokens": generated_tokens, + "total_tokens": total_tokens, + } + if latencies: + metrics["response_latency_s"] = round(sum(latencies), 6) + if ttfts: + metrics["ttft_ms"] = round(1000.0 * sum(ttfts) / len(ttfts), 6) + if tbts: + metrics["tbt_ms"] = round(1000.0 * sum(tbts) / len(tbts), 6) + return metrics + + +def _configured_requests(measurement: Mapping[str, Any]) -> List[Mapping[str, Any]]: + requests = ( + measurement.get("requests") + or measurement.get("sglang_requests") + or measurement.get("openai_requests") + or [] + ) + if not isinstance(requests, list): + raise SGLangMeasurementError("measurement.requests must be a list") + for request in requests: + if not isinstance(request, dict): + raise SGLangMeasurementError("each measurement request must be a mapping") + return requests + + +def _run_configured_requests( + base_url: str, + measurement: Mapping[str, Any], + events: List[Dict[str, Any]], +) -> None: + request_timeout = float(measurement.get("request_timeout_seconds", 120)) + for index, item in enumerate(_configured_requests(measurement)): + endpoint = str(item.get("endpoint", "/generate")) + if not endpoint.startswith("/"): + endpoint = "/" + endpoint + payload = item.get("payload", {}) + if not isinstance(payload, dict): + raise SGLangMeasurementError("measurement.requests[%s].payload must be a mapping" % index) + started = time.perf_counter() + status, response = _post_json(base_url + endpoint, payload, request_timeout) + latency = time.perf_counter() - started + metrics = _extract_response_metrics(response) + event = { + "type": "request", + "backend": "sglang", + "request_id": index, + "endpoint": endpoint, + "status": status, + "latency_s": round(latency, 6), + "response_keys": sorted(response.keys()), + } + event.update(metrics) + events.append(event) + + +def _request_summary(events: List[Dict[str, Any]]) -> Dict[str, Any]: + request_events = [event for event in events if event.get("type") == "request"] + prompt_tokens = sum(_as_int(event.get("prompt_tokens")) for event in request_events) + generated_tokens = sum(_as_int(event.get("generated_tokens")) for event in request_events) + request_latency = sum(_as_float(event.get("latency_s")) for event in request_events) + ttft_values = [_as_float(event.get("ttft_ms")) for event in request_events if _as_float(event.get("ttft_ms")) > 0] + tbt_values = [_as_float(event.get("tbt_ms")) for event in request_events if _as_float(event.get("tbt_ms")) > 0] + return { + "prompt_tokens": prompt_tokens, + "generated_tokens": generated_tokens, + "request_latency_s": request_latency, + "ttft_ms": round(sum(ttft_values) / len(ttft_values), 6) if ttft_values else 0.0, + "tbt_ms": round(sum(tbt_values) / len(tbt_values), 6) if tbt_values else 0.0, + } + + def run_sglang(config: Mapping[str, Any], out_dir: str) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: _require_sglang() output_dir = Path(out_dir) @@ -134,6 +307,7 @@ def run_sglang(config: Mapping[str, Any], out_dir: str) -> Tuple[Dict[str, Any], events: List[Dict[str, Any]] = [{"type": "launch", "command": command}] sampler = _NvmlSampler(sample_hz, events) if nvml_enabled else None + collection_elapsed = duration with log_path.open("w", encoding="utf-8") as log_handle: process = subprocess.Popen(command, stdout=log_handle, stderr=subprocess.STDOUT) try: @@ -143,8 +317,12 @@ def run_sglang(config: Mapping[str, Any], out_dir: str) -> Tuple[Dict[str, Any], events.append({"type": "health", "url": health_url, "ok": True}) if sampler is not None: sampler.start() + collection_start = time.monotonic() + configured_requests = _configured_requests(measurement) workload_command = measurement.get("workload_command") - if workload_command: + if configured_requests: + _run_configured_requests(_server_base_url(config), measurement, events) + elif workload_command: if not isinstance(workload_command, list): raise SGLangMeasurementError("measurement.workload_command must be a list of command arguments") workload = subprocess.run(workload_command, check=False, capture_output=True, text=True) @@ -159,6 +337,7 @@ def run_sglang(config: Mapping[str, Any], out_dir: str) -> Tuple[Dict[str, Any], else: time.sleep(duration) events.append({"type": "idle_collection", "duration_seconds": duration}) + collection_elapsed = max(time.monotonic() - collection_start, 0.000001) finally: sampler_error = None if sampler is not None: @@ -174,20 +353,36 @@ def run_sglang(config: Mapping[str, Any], out_dir: str) -> Tuple[Dict[str, Any], if sampler_error is not None: raise sampler_error - elapsed, avg_power, energy = _power_summary(events, duration) + requests = _request_summary(events) + fallback_elapsed = max(collection_elapsed, requests["request_latency_s"], duration if not requests["generated_tokens"] else 0.000001) + elapsed, avg_power, energy = _power_summary(events, fallback_elapsed) + generated_tokens = int(requests["generated_tokens"]) + tokens_per_second = formulas.safe_div(generated_tokens, elapsed) + tokens_per_watt = formulas.safe_div(tokens_per_second, avg_power) + tokens_per_joule = formulas.safe_div(generated_tokens, energy) + quality = float((config.get("model", {}) or {}).get("quality_score", 1.0)) row = { "experiment": (config.get("experiment", {}) or {}).get("name", "aether-sglang"), "backend": "sglang", "scenario_id": "sglang-real", "model": (config.get("model", {}) or {}).get("name", "unknown-model"), - "hardware": "real-gpu", + "hardware": "real-sglang", "routing_policy": "real", "routing_pool": "sglang", "scheduling_policy": "real", "scheduling_action": "observed", + "prompt_tokens": int(requests["prompt_tokens"]), + "generated_tokens": generated_tokens, "elapsed_seconds": round(elapsed, 6), "energy_j": round(energy, 6), "avg_power_w": round(avg_power, 6), + "tokens_per_second": round(tokens_per_second, 6), + "tokens_per_watt": round(tokens_per_watt, 9), + "tokens_per_joule": round(tokens_per_joule, 9), + "quality_score": quality, + "quality_normalized_ipw": round(quality * tokens_per_joule, 9), + "ttft_ms": requests["ttft_ms"], + "tbt_ms": requests["tbt_ms"], } write_csv([row], str(output_dir / "summary.csv")) write_jsonl(events, str(output_dir / "events.jsonl")) diff --git a/tests/test_sglang_measurement.py b/tests/test_sglang_measurement.py new file mode 100644 index 0000000..3a7574e --- /dev/null +++ b/tests/test_sglang_measurement.py @@ -0,0 +1,88 @@ +import aether.measurement.sglang as sglang_measurement +from aether.measurement.sglang import ( + _extract_response_metrics, + _request_summary, + _run_configured_requests, +) + + +def test_extract_response_metrics_from_sglang_generate_response(): + metrics = _extract_response_metrics( + { + "text": "ok", + "meta_info": { + "prompt_tokens": 5, + "completion_tokens": 3, + "e2e_latency": 0.25, + "ttft": 0.04, + "tbt": 0.01, + }, + } + ) + + assert metrics["prompt_tokens"] == 5 + assert metrics["generated_tokens"] == 3 + assert metrics["total_tokens"] == 8 + assert metrics["response_latency_s"] == 0.25 + assert metrics["ttft_ms"] == 40.0 + assert metrics["tbt_ms"] == 10.0 + + +def test_extract_response_metrics_from_openai_usage_response(): + metrics = _extract_response_metrics( + { + "choices": [{"text": "ok"}], + "usage": {"prompt_tokens": 4, "completion_tokens": 2, "total_tokens": 6}, + } + ) + + assert metrics == {"prompt_tokens": 4, "generated_tokens": 2, "total_tokens": 6} + + +def test_run_configured_requests_posts_json_and_records_metrics(monkeypatch): + calls = [] + + def fake_post_json(url, payload, timeout_s): + calls.append((url, payload, timeout_s)) + return ( + 200, + { + "text": "hi", + "meta_info": { + "prompt_tokens": 7, + "completion_tokens": 2, + "e2e_latency": 0.5, + }, + }, + ) + + monkeypatch.setattr(sglang_measurement, "_post_json", fake_post_json) + events = [] + _run_configured_requests( + "http://127.0.0.1:30080", + { + "requests": [ + { + "endpoint": "/generate", + "payload": { + "text": "hello", + "sampling_params": {"temperature": 0.0, "max_new_tokens": 2}, + }, + } + ] + }, + events, + ) + + assert calls == [ + ( + "http://127.0.0.1:30080/generate", + {"text": "hello", "sampling_params": {"temperature": 0.0, "max_new_tokens": 2}}, + 120.0, + ) + ] + assert events[0]["type"] == "request" + assert events[0]["status"] == 200 + assert events[0]["prompt_tokens"] == 7 + assert events[0]["generated_tokens"] == 2 + assert _request_summary(events)["generated_tokens"] == 2