From e41b37ed954f805039e1f171a76d56aca1708f85 Mon Sep 17 00:00:00 2001 From: David Stone Date: Tue, 24 Mar 2026 21:51:59 -0600 Subject: [PATCH 1/2] fix: install Composer deps before WP Performance Metrics CI job The swissspidy/wp-performance-action@v2 mounts the plugin directory into WordPress Playground. Without Composer dependencies, the plugin's autoloader is missing and the plugin fails to initialize, causing the Playwright global setup (requestUtils.setupRest()) to time out and exit with code 1. Changes: - Add shivammathur/setup-php@v2 step before the performance test - Run composer install --no-dev to build the vendor/ directory - Pass debug: runner.debug to enable verbose output when re-run with debug The job retains continue-on-error: true since performance metrics are informational and transient infrastructure failures should not block CI. Closes #435 --- .github/workflows/tests.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fb53ece87..ebd699f6f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -103,6 +103,14 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: "8.2" + + - name: Install PHP Dependencies + run: composer install --no-dev --no-interaction --prefer-dist --optimize-autoloader + - name: Run WP Performance Tests uses: swissspidy/wp-performance-action@v2 with: @@ -113,3 +121,4 @@ jobs: create-comment: ${{ github.event_name == 'pull_request' }} print-results: true upload-artifacts: true + debug: ${{ runner.debug == '1' }} From 10209c4a513991a3658fe715eac8e2c87ffba5ea Mon Sep 17 00:00:00 2001 From: David Stone Date: Tue, 24 Mar 2026 21:55:09 -0600 Subject: [PATCH 2/2] fix: add noise floor to performance comparator to prevent false positives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Performance Benchmark workflow compares usleep()-based timings between two CI runner instances. Sub-millisecond measurements (e.g. 0.26ms vs 0.37ms) are dominated by scheduler jitter and CPU load variance — not real regressions. A 42% change on 0.26ms is only 0.11ms of absolute difference, which is well within normal system noise on shared CI runners. This caused the benchmark to flag false critical regressions on PRs that made no PHP changes at all. Add a minimum_absolute_change map that gates percentage threshold checks: - execution_time_ms: require at least 1ms absolute change before flagging - memory_usage_mb: require at least 0.5MB absolute change - database_queries: require at least 1 query absolute change This preserves detection of real regressions (e.g. a function that goes from 5ms to 8ms is a real +60% regression worth flagging) while ignoring noise in sub-millisecond usleep() simulations. --- tests/performance-comparator.php | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/performance-comparator.php b/tests/performance-comparator.php index c731e4354..deae90dd9 100755 --- a/tests/performance-comparator.php +++ b/tests/performance-comparator.php @@ -22,6 +22,18 @@ class Performance_Comparator { 'memory_usage_mb' => 40, // 40% increase for critical 'database_queries' => 25, // 25% increase for critical ]; + + /** + * Minimum absolute change required before a percentage threshold is applied. + * Prevents false positives on sub-millisecond measurements where CI runner + * noise (scheduler jitter, CPU load) dominates over real regressions. + * A 42% change on 0.26ms is only 0.11ms — well within normal system noise. + */ + private $minimum_absolute_change = [ + 'execution_time_ms' => 1.0, // Ignore changes smaller than 1ms + 'memory_usage_mb' => 0.5, // Ignore changes smaller than 0.5MB + 'database_queries' => 1, // Ignore changes smaller than 1 query + ]; public function __construct($baseline_file, $current_file) { $this->baseline_results = $this->load_results($baseline_file); @@ -118,13 +130,24 @@ private function compare_benchmark($benchmark) { $change_percent = (($current_value - $baseline_value) / $baseline_value) * 100; + $absolute_change = $current_value - $baseline_value; + $result['changes'][$metric] = [ 'baseline' => $baseline_value, 'current' => $current_value, 'change_percent' => round($change_percent, 2), - 'change_absolute' => $current_value - $baseline_value + 'change_absolute' => $absolute_change ]; - + + // Skip threshold checks when the absolute change is below the noise floor. + // Sub-millisecond timing differences are dominated by CI runner jitter and + // do not represent real regressions. + $min_absolute = $this->minimum_absolute_change[$metric] ?? 0; + if (abs($absolute_change) < $min_absolute) { + // Treat as no_change — noise floor not exceeded + continue; + } + // Check for critical regression if ($change_percent > $this->critical_thresholds[$metric]) { $result['status'] = 'critical_regression';