feat: emit pacing_rate and ssthresh in qlog
#2663
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance comparison | |
| on: | |
| workflow_call: | |
| pull_request: | |
| workflow_dispatch: | |
| env: | |
| CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true | |
| CARGO_PROFILE_RELEASE_DEBUG: true | |
| RUSTUP_TOOLCHAIN: stable | |
| SCCACHE_CACHE_SIZE: 128G | |
| SCCACHE_DIRECT: true | |
| MTU: 1504 # https://github.com/microsoft/msquic/issues/4618 | |
| CFLAGS: -fno-omit-frame-pointer | |
| CXXFLAGS: -fno-omit-frame-pointer | |
| BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| build-neqo: | |
| name: Build neqo | |
| runs-on: ubuntu-24.04 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout mozilla/neqo | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| repository: mozilla/neqo | |
| path: neqo | |
| submodules: 'recursive' | |
| persist-credentials: false | |
| clean: false | |
| - uses: ./neqo/.github/actions/build-neqo | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| artifact-name: build-neqo | |
| baseline-ref: origin/${{ env.BASE_REF }} | |
| build-msquic: | |
| name: Build msquic | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| sparse-checkout: .github/actions | |
| persist-credentials: false | |
| - uses: ./.github/actions/quic-build | |
| with: | |
| impl: msquic | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| artifact-name: build-msquic | |
| build-google: | |
| name: Build google/quiche | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| sparse-checkout: .github/actions | |
| persist-credentials: false | |
| - uses: ./.github/actions/quic-build | |
| with: | |
| impl: google | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| artifact-name: build-google | |
| build-quiche: | |
| name: Build cloudflare/quiche | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| sparse-checkout: .github/actions | |
| persist-credentials: false | |
| - uses: ./.github/actions/quic-build | |
| with: | |
| impl: quiche | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| artifact-name: build-quiche | |
| build-s2n: | |
| name: Build s2n-quic | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| sparse-checkout: .github/actions | |
| persist-credentials: false | |
| - uses: ./.github/actions/quic-build | |
| with: | |
| impl: s2n | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| artifact-name: build-s2n | |
| perfcompare: | |
| name: Performance comparison | |
| runs-on: 'self-hosted' # zizmor: ignore[self-hosted-runner] | |
| needs: [build-neqo, build-msquic, build-google, build-quiche, build-s2n] | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout mozilla/neqo | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| repository: mozilla/neqo | |
| path: neqo | |
| persist-credentials: false | |
| clean: false | |
| - run: | | |
| cd neqo | |
| git fetch --no-tags --depth=1 origin "$BASE_REF" | |
| - name: Set PATH and environment | |
| run: | | |
| echo "$HOME/.cargo/bin" >> "${GITHUB_PATH}" | |
| - name: Install Rust | |
| uses: ./neqo/.github/actions/rust | |
| with: | |
| version: ${{ env.RUSTUP_TOOLCHAIN }} | |
| tools: hyperfine, flamegraph, samply | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| with: | |
| name: build-neqo | |
| path: build-neqo | |
| - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| with: | |
| name: build-msquic | |
| path: build-msquic | |
| - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| with: | |
| name: build-google | |
| path: build-google | |
| - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| with: | |
| name: build-quiche | |
| path: build-quiche | |
| - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 | |
| with: | |
| name: build-s2n | |
| path: build-s2n | |
| - name: Set up build artifacts | |
| run: | | |
| # Clean stale binaries dir from previous workflow runs. | |
| # perfcompare.py recreates and populates binaries/ during the run. | |
| rm -rf binaries | |
| chmod +x build-neqo/neqo/* build-neqo/neqo-baseline/* \ | |
| build-msquic/* build-google/* build-quiche/* build-s2n/* | |
| { | |
| echo "LD_LIBRARY_PATH=$GITHUB_WORKSPACE/build-neqo/lib:$LD_LIBRARY_PATH" | |
| echo "NSS_DB_PATH=$GITHUB_WORKSPACE/build-neqo/test-fixture/db" | |
| } >> "$GITHUB_ENV" | |
| # Disable turboboost, hyperthreading and use performance governor. | |
| # Also creates "cpu23", "cpu2" and "cpu3" CPU sets for use with cset. | |
| # On the bencher, logical cores 2 and 3 have been isolated for use by the benchmarks. | |
| - name: Prepare machine | |
| run: | | |
| sudo /root/bin/prep.sh | |
| # See https://github.com/microsoft/msquic/issues/4618#issuecomment-2422611592 | |
| sudo ip link set dev lo mtu "$MTU" | |
| # Compare various configurations of neqo against msquic and google/quiche, and gather perf data | |
| # during the hyperfine runs. | |
| - name: Compare QUIC implementations | |
| env: | |
| WORKSPACE: ${{ github.workspace }} | |
| NSS_DB_PATH: ${{ github.workspace }}/build-neqo/test-fixture/db | |
| run: | | |
| python3 neqo/.github/scripts/perfcompare.py \ | |
| --host 127.0.0.1 \ | |
| --port 4433 \ | |
| --size 33554432 \ | |
| --runs 100 \ | |
| --workspace "$WORKSPACE" \ | |
| --perf-opt "record -F2999 --call-graph fp -g" | |
| # Re-enable turboboost, hyperthreading and use powersave governor. Remove all CPU sets. | |
| - name: Restore machine | |
| if: success() || failure() || cancelled() | |
| run: | | |
| sudo /root/bin/unprep.sh | |
| # In case the previous test failed: | |
| sudo ip link set dev lo mtu 65536 | |
| - name: Post-process perf data | |
| run: | | |
| for f in *.perf; do | |
| # Convert for profiler.firefox.com | |
| samply import "$f" -o "$f.samply.json.gz" --save-only --presymbolicate | |
| # Generate flamegraphs | |
| flamegraph --perfdata "$f" --palette rust -o "${f//.perf/.svg}" | |
| done | |
| - name: Format results as Markdown | |
| id: results | |
| env: | |
| EVENT_PATH: ${{ github.event_path }} | |
| TESTBED: 'On-prem' | |
| run: | | |
| SHA=$(cd neqo && git log origin/"$BASE_REF" -1 --format=%H) | |
| echo "$SHA" > baseline-sha.txt | |
| { | |
| echo "### Client/server transfer results" | |
| echo | |
| echo "Performance differences relative to $SHA." | |
| echo | |
| # Extract rows with significant changes (red or green heart). | |
| if grep -q ':broken_heart:\|:green_heart:' comparison.md; then | |
| # Print all lines up to and including the first separator line (header) | |
| awk 'BEGIN{found=0} /^[|][ -|:]+[|]$/ {found=1} {print} found{exit}' comparison.md | |
| grep ':broken_heart:\|:green_heart:' comparison.md | |
| echo | |
| else | |
| echo "No significant performance differences." | |
| echo | |
| fi | |
| echo "Table above only shows statistically significant changes. See all results below." | |
| echo "<details><summary>All results</summary>" | |
| echo | |
| cat comparison.md | |
| echo | |
| echo "</details>" | |
| } >> results.md | |
| cat results.md > "$GITHUB_STEP_SUMMARY" | |
| echo "$TESTBED" > testbed.txt | |
| cp "$EVENT_PATH" event.json | |
| - name: Export profiler.firefox.com data | |
| id: export_samply | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-perfcompare-samply | |
| path: | | |
| *.samply.json.gz | |
| binaries | |
| compression-level: 9 | |
| - name: Export performance data | |
| id: export_perf | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: ${{ github.event.repository.name }}-${{ github.event.pull_request.head.sha }}-perfcompare-perf | |
| path: | | |
| *.svg | |
| *.txt | |
| *.md | |
| event.json | |
| results.* | |
| results-baseline.* | |
| hyperfine | |
| hyperfine-baseline | |
| compression-level: 9 | |
| - name: Export PR comment data | |
| uses: ./neqo/.github/actions/pr-comment-data-export | |
| with: | |
| name: ${{ github.workflow }} | |
| contents: results.md | |
| log-md: ${{ format('[Download data for `profiler.firefox.com`]({0}) or [download performance comparison data]({1}).', steps.export_samply.outputs.artifact-url, steps.export_perf.outputs.artifact-url) }} | |
| - name: Fail on regression | |
| # Don't check for regressions when running on main. | |
| if: github.ref != 'refs/heads/main' && github.event.merge_group.base_ref != 'refs/heads/main' | |
| run: | | |
| if grep -q "Performance has regressed." results.txt; then | |
| echo "Performance regression detected." | |
| exit 1 | |
| else | |
| echo "No performance regression detected." | |
| fi | |
| - name: Remove benchmark artifacts | |
| if: always() | |
| run: | | |
| rm -- * || true | |
| rm -r -- binaries build-neqo build-msquic build-google build-quiche build-s2n \ | |
| comment-data hyperfine hyperfine-baseline || true |