LOGBROKER-10206 Refactor writers #5522
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: SLO | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled] | |
| branches: | |
| - master | |
| - release-* | |
| workflow_dispatch: | |
| inputs: | |
| github_issue: | |
| description: "GitHub issue number where the SLO results will be reported" | |
| required: true | |
| baseline_ref: | |
| description: "Baseline commit/branch/tag to compare against (leave empty to auto-detect merge-base with main)" | |
| required: false | |
| slo_workload_duration_seconds: | |
| description: "Duration of the SLO workload in seconds" | |
| required: false | |
| default: "600" | |
| slo_workload_read_max_rps: | |
| description: "Maximum read RPS for the SLO workload" | |
| required: false | |
| default: "1000" | |
| slo_workload_write_max_rps: | |
| description: "Maximum write RPS for the SLO workload" | |
| required: false | |
| default: "100" | |
| jobs: | |
| ydb-slo-action: | |
| if: contains(github.event.pull_request.labels.*.name, 'SLO') | |
| name: Run YDB SLO Tests | |
| runs-on: "large-runner" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| sdk: | |
| - id: database_sql_table | |
| name: database-sql-table | |
| path: ./database/sql/table | |
| label: database/sql/table | |
| run_extra_args: '' | |
| create_extra_args: '' | |
| - id: database_sql_query | |
| name: database-sql-query | |
| path: ./database/sql/query | |
| label: database/sql/query | |
| run_extra_args: '' | |
| create_extra_args: '' | |
| - id: native_query | |
| name: native-query | |
| path: ./native/query | |
| label: native/query | |
| run_extra_args: '' | |
| create_extra_args: '' | |
| - id: native_table | |
| name: native-table | |
| path: ./native/table | |
| label: native/table | |
| run_extra_args: '' | |
| create_extra_args: '' | |
| - id: native_table_over_query_service | |
| name: native-table-over-query-service | |
| path: ./native/table/over/query/service | |
| label: native/table/over/query/service | |
| run_extra_args: '' | |
| create_extra_args: '' | |
| - id: native_bulk_upsert | |
| name: native-bulk-upsert | |
| path: ./native/bulk-upsert | |
| label: native/bulk-upsert | |
| run_extra_args: '-batch-size=10' | |
| create_extra_args: '' | |
| - id: native_node_hints | |
| name: native-node-hints | |
| path: ./native/node_hints | |
| label: native/node_hints | |
| slo_workload_read_max_rps: 100 | |
| slo_workload_write_max_rps: 100 | |
| run_extra_args: '-batch-size=10' | |
| create_extra_args: '-min-partitions-count 10' | |
| - id: session_node_hints | |
| name: session-node-hints | |
| path: ./native/session_node_hints | |
| label: native/session_node_hints | |
| slo_workload_read_max_rps: 1000 | |
| slo_workload_write_max_rps: 1 | |
| run_extra_args: '-batch-size=10' | |
| create_extra_args: '-min-partitions-count 10' | |
| concurrency: | |
| group: slo-${{ github.ref }}-${{ matrix.sdk.name }} | |
| cancel-in-progress: true | |
| steps: | |
| - name: Install dependencies | |
| run: | | |
| YQ_VERSION=v4.48.2 | |
| BUILDX_VERSION=0.30.1 | |
| COMPOSE_VERSION=2.40.3 | |
| sudo curl -L https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -o /usr/local/bin/yq && \ | |
| sudo chmod +x /usr/local/bin/yq | |
| echo "Updating Docker plugins..." | |
| sudo mkdir -p /usr/local/lib/docker/cli-plugins | |
| echo "Installing Docker Buildx ${BUILDX_VERSION}..." | |
| sudo curl -fLo /usr/local/lib/docker/cli-plugins/docker-buildx \ | |
| "https://github.com/docker/buildx/releases/download/v${BUILDX_VERSION}/buildx-v${BUILDX_VERSION}.linux-amd64" | |
| sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx | |
| echo "Installing Docker Compose ${COMPOSE_VERSION}..." | |
| sudo curl -fLo /usr/local/lib/docker/cli-plugins/docker-compose \ | |
| "https://github.com/docker/compose/releases/download/v${COMPOSE_VERSION}/docker-compose-linux-x86_64" | |
| sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose | |
| echo "Installed versions:" | |
| yq --version | |
| docker --version | |
| docker buildx version | |
| docker compose version | |
| - name: Checkout current version | |
| uses: actions/checkout@v5 | |
| with: | |
| path: current | |
| fetch-depth: 0 | |
| - name: Determine baseline commit | |
| id: baseline | |
| run: | | |
| cd current | |
| if [[ -n "${{ inputs.baseline_ref }}" ]]; then | |
| BASELINE="${{ inputs.baseline_ref }}" | |
| else | |
| BASELINE=$(git merge-base HEAD origin/master) | |
| fi | |
| echo "sha=$BASELINE" >> $GITHUB_OUTPUT | |
| # Try to determine a human-readable ref name for baseline | |
| # Check if baseline is on master | |
| if git merge-base --is-ancestor $BASELINE origin/master && \ | |
| [ "$(git rev-parse origin/master)" = "$BASELINE" ]; then | |
| BASELINE_REF="master" | |
| else | |
| # Try to find a branch containing this commit | |
| BRANCH=$(git branch -r --contains $BASELINE | grep -v HEAD | head -1 | sed 's/.*\///' || echo "") | |
| if [ -n "$BRANCH" ]; then | |
| BASELINE_REF="${BRANCH}@${BASELINE:0:7}" | |
| else | |
| BASELINE_REF="${BASELINE:0:7}" | |
| fi | |
| fi | |
| echo "ref=$BASELINE_REF" >> $GITHUB_OUTPUT | |
| - name: Checkout baseline version | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ steps.baseline.outputs.sha }} | |
| path: baseline | |
| fetch-depth: 1 | |
| - name: Build workload images | |
| run: | | |
| SCRIPT="$GITHUB_WORKSPACE/current/.github/scripts/build-slo-image.sh" | |
| CURRENT_REF="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | |
| bash "$SCRIPT" \ | |
| --context "$GITHUB_WORKSPACE/current" \ | |
| --tag "ydb-app-current" \ | |
| --ref "${CURRENT_REF}" \ | |
| --src-path "${{ matrix.sdk.label }}" \ | |
| --job-name "${{ matrix.sdk.name }}" | |
| bash "$SCRIPT" \ | |
| --context "$GITHUB_WORKSPACE/baseline" \ | |
| --tag "ydb-app-baseline" \ | |
| --ref "${{ steps.baseline.outputs.ref }}" \ | |
| --src-path "${{ matrix.sdk.label }}" \ | |
| --job-name "${{ matrix.sdk.name }}" \ | |
| --fallback-image "ydb-app-current" | |
| - name: Initialize YDB SLO | |
| id: ydb_slo | |
| uses: ydb-platform/ydb-slo-action/init@13c687b7d4b2879da79dd12932dee0ed2b65dd1c | |
| with: | |
| github_issue: ${{ github.event.inputs.github_issue }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| workload_name: ${{ matrix.sdk.name }} | |
| workload_current_ref: ${{ github.head_ref || github.ref_name }} | |
| workload_baseline_ref: ${{ steps.baseline.outputs.ref }} | |
| # to disable chaos testing for node hint workloads | |
| disable_compose_profiles: "${{ (matrix.sdk.id == 'native_node_hints' || matrix.sdk.id == 'session_node_hints') && 'chaos' || '' }}" | |
| metrics_yaml: | | |
| metrics: | |
| - name: node_hints_misses | |
| type: instant | |
| query: sum by(ref) (workload_node_hints_misses) | |
| - name: Prepare SLO Database | |
| run: | | |
| echo "Preparing SLO database..." | |
| CREATE_EXTRA_ARGS="${{ matrix.sdk.create_extra_args }}" | |
| docker run --rm --network ydb_ydb-net \ | |
| --add-host "ydb:172.28.0.11" \ | |
| --add-host "ydb:172.28.0.12" \ | |
| --add-host "ydb:172.28.0.13" \ | |
| --add-host "ydb:172.28.0.99" \ | |
| ydb-app-current create grpc://ydb:2136 /Root/testdb $CREATE_EXTRA_ARGS | |
| - name: Run SLO Tests (parallel) | |
| timeout-minutes: 15 | |
| env: | |
| DURATION: ${{ matrix.sdk.slo_workload_duration_seconds || 600 }} | |
| READ_RPS: ${{ matrix.sdk.slo_workload_read_max_rps || 1000 }} | |
| WRITE_RPS: ${{ matrix.sdk.slo_workload_write_max_rps || 1000 }} | |
| RUN_EXTRA_ARGS: ${{ format('{0} -prometheus-endpoint {1}',matrix.sdk.run_extra_args,steps.ydb_slo.outputs.ydb-prometheus-url)}} | |
| run: | | |
| ARGS="run grpc://ydb:2136 /Root/testdb \ | |
| -otlp-endpoint prometheus:9090 \ | |
| -report-period 250 \ | |
| -time $DURATION \ | |
| -read-rps $READ_RPS \ | |
| -write-rps $WRITE_RPS \ | |
| -read-timeout 100 \ | |
| -write-timeout 100 \ | |
| $RUN_EXTRA_ARGS" | |
| echo "Starting ydb-app-current..." | |
| docker run -d \ | |
| --name ydb-app-current \ | |
| --network ydb_ydb-net \ | |
| --add-host "ydb:172.28.0.11" \ | |
| --add-host "ydb:172.28.0.12" \ | |
| --add-host "ydb:172.28.0.13" \ | |
| --add-host "ydb:172.28.0.99" \ | |
| ydb-app-current $ARGS | |
| echo "Starting ydb-app-baseline..." | |
| docker run -d \ | |
| --name ydb-app-baseline \ | |
| --network ydb_ydb-net \ | |
| --add-host "ydb:172.28.0.11" \ | |
| --add-host "ydb:172.28.0.12" \ | |
| --add-host "ydb:172.28.0.13" \ | |
| --add-host "ydb:172.28.0.99" \ | |
| ydb-app-baseline $ARGS | |
| # Show initial logs | |
| echo "" | |
| echo "==================== INITIAL CURRENT LOGS ====================" | |
| docker logs -n 15 ydb-app-current 2>&1 || echo "No current container" | |
| echo "" | |
| echo "==================== INITIAL BASELINE LOGS ====================" | |
| docker logs -n 15 ydb-app-baseline 2>&1 || echo "No baseline container" | |
| echo "" | |
| # Wait for workloads to complete | |
| echo "Waiting for workloads to complete (${DURATION}s)..." | |
| sleep ${DURATION} | |
| # Stop containers after workload duration and wait for graceful shutdown | |
| echo "Stopping containers after ${DURATION}s..." | |
| docker stop --timeout=30 ydb-app-current ydb-app-baseline 2>&1 || true | |
| # Force kill if still running | |
| docker kill ydb-app-current ydb-app-baseline 2>&1 || true | |
| # Check exit codes | |
| CURRENT_EXIT=$(docker inspect ydb-app-current --format='{{.State.ExitCode}}' 2>/dev/null || echo "1") | |
| BASELINE_EXIT=$(docker inspect ydb-app-baseline --format='{{.State.ExitCode}}' 2>/dev/null || echo "0") | |
| echo "Current container exit code: $CURRENT_EXIT" | |
| echo "Baseline container exit code: $BASELINE_EXIT" | |
| # Show final logs | |
| echo "" | |
| echo "==================== FINAL CURRENT LOGS ====================" | |
| docker logs -n 15 ydb-app-current 2>&1 || echo "No current container" | |
| echo "" | |
| echo "==================== FINAL BASELINE LOGS ====================" | |
| docker logs -n 15 ydb-app-baseline 2>&1 || echo "No baseline container" | |
| echo "" | |
| echo "SUCCESS: Workloads completed successfully" | |
| - if: always() | |
| name: Store logs | |
| run: | | |
| docker logs ydb-app-current > current.log 2>&1 || echo "No current container" | |
| docker logs ydb-app-baseline > baseline.log 2>&1 || echo "No baseline container" | |
| - if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{matrix.sdk.name}}-logs | |
| path: | | |
| ./current.log | |
| ./baseline.log | |
| retention-days: 1 |