From a27bcf9e42e3b1a483e2eba74871206d2a7bb422 Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Fri, 29 Aug 2025 17:19:06 +0200
Subject: [PATCH 01/13] ci: add flaky tests bot workflow

---
 .github/scripts/create-flaky-test-report.mjs | 435 +++++++++++++++++++
 1 file changed, 435 insertions(+)
 create mode 100644 .github/scripts/create-flaky-test-report.mjs

diff --git a/.github/scripts/create-flaky-test-report.mjs b/.github/scripts/create-flaky-test-report.mjs
new file mode 100644
index 00000000..84b82509
--- /dev/null
+++ b/.github/scripts/create-flaky-test-report.mjs
@@ -0,0 +1,435 @@
+#!/usr/bin/env node
+
+import { Octokit } from 'octokit';
+import unzipper from 'unzipper';
+
+const githubToken = process.env.GITHUB_TOKEN;
+if (!githubToken) throw new Error('Missing GITHUB_TOKEN env var');
+
+const env = {
+  OWNER: process.env.OWNER || 'MetaMask',
+  REPOSITORY: process.env.REPOSITORY || 'metamask-extension',
+  WORKFLOW_ID: process.env.WORKFLOW_ID || 'main.yml',
+  BRANCH: process.env.BRANCH || 'main',
+  GITHUB_TOKEN: process.env.GITHUB_TOKEN,
+  LOOKBACK_DAYS: parseInt(process.env.LOOKBACK_DAYS ?? '1'),
+  TEST_REPORT_ARTIFACTS: process.env.TEST_REPORT_ARTIFACTS
+    ? process.env.TEST_REPORT_ARTIFACTS.split(',').map(name => name.trim())
+    : ['test-e2e-chrome-report', 'test-e2e-firefox-report'],
+  TEST_RESULTS_FILE_PATTERN: process.env.TEST_RESULTS_FILE_PATTERN || 'test-runs',
+  // For mobile, you can override these:
+  // REPOSITORY=metamask-mobile TEST_REPORT_ARTIFACTS=test-e2e-android-report,test-e2e-ios-report
+};
+
+function getDateRange() {
+  const today = new Date();
+  const daysAgo = new Date(today.getTime() - (env.LOOKBACK_DAYS * 24 * 60 * 60 * 1000));
+
+  const fromDisplay = daysAgo.toLocaleDateString('en-US', {
+    weekday: 'short',
+    month: 'short',
+    day: 'numeric',
+    hour: 'numeric',
+    minute: '2-digit'
+  });
+
+  const toDisplay = today.toLocaleDateString('en-US', {
+    weekday: 'short',
+    month: 'short',
+    day: 'numeric',
+    hour: 'numeric',
+    minute: '2-digit'
+  });
+
+  return {
+    from: daysAgo.toISOString(),
+    to: today.toISOString(),
+    display: `${fromDisplay} - ${toDisplay}`
+  };
+}
+
+async function getFailedWorkflowRuns(github, from, to) {
+  try {
+    const runs = await github.paginate(
+      github.rest.actions.listWorkflowRuns,
+      {
+        owner: env.OWNER,
+        repo: env.REPOSITORY,
+        workflow_id: env.WORKFLOW_ID,
+        branch: env.BRANCH,
+        status: 'failure',
+        created: `${from}..${to}`,
+        per_page: 100,
+      }
+    );
+
+    return runs;
+  } catch (error) {
+    if (error.status === 404) {
+      throw new Error(`Workflow '${env.WORKFLOW_ID}' not found in ${env.OWNER}/${env.REPOSITORY}`);
+    }
+    throw error;
+  }
+}
+
+async function downloadArtifact(github, artifact) {
+  try {
+    const response = await github.rest.actions.downloadArtifact({
+      owner: env.OWNER,
+      repo: env.REPOSITORY,
+      artifact_id: artifact.id,
+      archive_format: 'zip',
+    });
+
+    const buffer = Buffer.from(response.data);
+    const zip = await unzipper.Open.buffer(buffer);
+
+    const testFile = zip.files.find(file => file.path.startsWith(env.TEST_RESULTS_FILE_PATTERN));
+    if (!testFile) {
+      console.log(`  ⚠️  No ${env.TEST_RESULTS_FILE_PATTERN} file found in ${artifact.name}`);
+      return null;
+    }
+
+    const content = await testFile.buffer();
+    const data = JSON.parse(content.toString());
+
+    console.log(`   Parsed ${artifact.name} (${data.length} top testSuites)`);
+    return data;
+  } catch (error) {
+    console.log(`  ❌ Failed to download ${artifact.name}: ${error.message}`);
+    return null;
+  }
+}
+
+async function downloadTestArtifacts(github, runs) {
+  const allTestData = [];
+
+  for (const [index, run] of runs.entries()) {
+    console.log(`📦 Processing run ${index + 1}/${runs.length}: ${run.head_commit?.message?.split('\n')[0] || 'No commit message'}`);
+
+    try {
+      const artifacts = await github.paginate(
+        github.rest.actions.listWorkflowRunArtifacts,
+        {
+          owner: env.OWNER,
+          repo: env.REPOSITORY,
+          run_id: run.id,
+        }
+      );
+
+      const testArtifacts = artifacts.filter(artifact =>
+        env.TEST_REPORT_ARTIFACTS.includes(artifact.name)
+      );
+
+      if (testArtifacts.length === 0) {
+        console.log(`  ⚠️  No test artifacts found for run ${run.id}`);
+        continue;
+      }
+
+      for (const artifact of testArtifacts) {
+        const testData = await downloadArtifact(github, artifact);
+        if (testData) {
+          allTestData.push(...testData);
+        }
+      }
+    } catch (error) {
+      console.log(`  ❌ Failed to process run ${run.id}: ${error.message}`);
+    }
+  }
+
+  return allTestData;
+}
+
+
+function extractRealFailures(testData) {
+  const realFailures = [];
+
+  for (const testRun of testData) {
+    for (const testFile of testRun.testFiles || []) {
+      for (const testSuite of testFile.testSuites || []) {
+        const retryCount = testSuite.attempts ? testSuite.attempts.length : 0;
+
+        // Process tests that failed even after retries
+        for (const testCase of testSuite.testCases || []) {
+          if (testCase.status === 'failed') {
+            realFailures.push({
+              name: testCase.name,
+              path: testFile.path,
+              error: testCase.error || 'No error details',
+              time: testCase.time || 0,
+              suite: testSuite.name,
+              jobId: testSuite.job?.id,
+              runId: testSuite.job?.runId,
+              date: new Date(testSuite.date || Date.now()),
+              retryCount: retryCount,
+              type: 'real_failure'
+            });
+          }
+        }
+      }
+    }
+  }
+
+  return realFailures;
+}
+
+function extractFlakyTests(testData) {
+  const flakyTests = [];
+
+  for (const testRun of testData) {
+    for (const testFile of testRun.testFiles || []) {
+      for (const testSuite of testFile.testSuites || []) {
+        const retryCount = testSuite.attempts ? testSuite.attempts.length : 0;
+
+        // Only process suites that have attempts (retries)
+        if (retryCount > 0) {
+          // Track failed tests in attempts
+          const failedInAttempts = new Map();
+          for (const attempt of testSuite.attempts || []) {
+            for (const testCase of attempt.testCases || []) {
+              if (testCase.status === 'failed') {
+                failedInAttempts.set(testCase.name, {
+                  jobId: attempt.job?.id,
+                  runId: attempt.job?.runId,
+                  error: testCase.error || 'No error details',
+                  date: new Date(attempt.date || Date.now())
+                });
+              }
+            }
+          }
+
+          // Process tests that eventually passed but had initial failures
+          for (const testCase of testSuite.testCases || []) {
+            if (testCase.status === 'passed' && failedInAttempts.has(testCase.name)) {
+              const failureInfo = failedInAttempts.get(testCase.name);
+              flakyTests.push({
+                name: testCase.name,
+                path: testFile.path,
+                error: failureInfo.error,
+                time: testCase.time || 0,
+                suite: testSuite.name,
+                jobId: failureInfo.jobId,
+                runId: failureInfo.runId,
+                date: failureInfo.date,
+                retryCount: retryCount,
+                type: 'flaky'
+              });
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return flakyTests;
+}
+
+
+function summarizeFailures(realFailures, flakyTests = []) {
+  const summary = new Map();
+
+  // Process real failures first
+  for (const test of realFailures) {
+    if (summary.has(test.name)) {
+      const existing = summary.get(test.name);
+      existing.realFailures += 1;
+      existing.totalRetries += test.retryCount;
+      // Update to chronologically latest real failure
+      if (test.date > existing.lastRealFailureDate) {
+        existing.lastRealFailureJobId = test.jobId;
+        existing.lastRealFailureRunId = test.runId;
+        existing.lastRealFailureError = test.error;
+        existing.lastRealFailureDate = test.date;
+      }
+      // Update last seen
+      if (test.date > existing.lastSeen) {
+        existing.lastSeen = test.date;
+      }
+    } else {
+      summary.set(test.name, {
+        name: test.name,
+        path: test.path,
+        realFailures: 1,
+        totalRetries: test.retryCount,
+        lastSeen: test.date,
+        suite: test.suite,
+        lastRealFailureJobId: test.jobId,
+        lastRealFailureRunId: test.runId,
+        lastRealFailureError: test.error,
+        lastRealFailureDate: test.date,
+        // Initialize flaky info as null
+        flakyFailureJobId: null,
+        flakyFailureRunId: null,
+        flakyFailureError: null,
+        flakyFailureDate: null
+      });
+    }
+  }
+
+  // Process flaky tests second
+  for (const test of flakyTests) {
+    if (summary.has(test.name)) {
+      // This test also had real failures - just add flaky info
+      const existing = summary.get(test.name);
+      existing.totalRetries += test.retryCount;
+      // Keep most recent flaky failure info
+      if (!existing.flakyFailureJobId || test.date > existing.flakyFailureDate) {
+        existing.flakyFailureJobId = test.jobId;
+        existing.flakyFailureRunId = test.runId;
+        existing.flakyFailureError = test.error;
+        existing.flakyFailureDate = test.date;
+      }
+      // Update last seen
+      if (test.date > existing.lastSeen) {
+        existing.lastSeen = test.date;
+      }
+    } else {
+      // This is purely a flaky test (no real failures)
+      summary.set(test.name, {
+        name: test.name,
+        path: test.path,
+        realFailures: 0,
+        totalRetries: test.retryCount,
+        lastSeen: test.date,
+        suite: test.suite,
+        // No real failure info
+        lastRealFailureJobId: null,
+        lastRealFailureRunId: null,
+        lastRealFailureError: null,
+        lastRealFailureDate: null,
+        // Flaky failure info
+        flakyFailureJobId: test.jobId,
+        flakyFailureRunId: test.runId,
+        flakyFailureError: test.error,
+        flakyFailureDate: test.date
+      });
+    }
+  }
+
+  return Array.from(summary.values())
+    .sort((a, b) => {
+      // Real failures first, sorted by failure count
+      if (a.realFailures !== b.realFailures) {
+        return b.realFailures - a.realFailures;
+      }
+      // If both have same real failure count, sort by total retries
+      return b.totalRetries - a.totalRetries;
+    });
+}
+
+function displayResults(summary, dateDisplay) {
+  console.log('\n' + '='.repeat(80));
+  console.log(`📊 FLAKY TEST REPORT - ${dateDisplay}`);
+  console.log('='.repeat(80));
+
+  if (summary.length === 0) {
+    console.log('\n✅ No failed tests found, great job!');
+    return;
+  }
+
+  const realFailures = summary.filter(test => test.realFailures > 0);
+  const flakyTests = summary.filter(test => test.realFailures === 0);
+
+  console.log(`${realFailures.length} real failures (failed even after retries)`);
+  console.log(`${flakyTests.length} flaky tests (eventually passed after retries)\n`);
+
+  const top10 = summary.slice(0, 10);
+
+  for (const [index, test] of top10.entries()) {
+    console.log(`${(index + 1).toString().padStart(2)}. ${test.name}`);
+    console.log(`    📁 File: ${test.path}`);
+
+    if (test.realFailures > 0) {
+      // Real failures (tests that failed even after retries)
+      const failurePlural = test.realFailures > 1 ? 's' : '';
+      const retryPlural = test.totalRetries > 1 ? 'retries' : 'retry';
+      const retryText = test.totalRetries > 0 ? ` (${test.totalRetries} ${retryPlural})` : '';
+      console.log(`    ❌ Failed: ${test.realFailures} time${failurePlural}${retryText}`);
+
+      // Show logs for real failures
+      if (test.lastRealFailureJobId && test.lastRealFailureRunId) {
+        console.log(`    🔗 Logs: https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${test.lastRealFailureRunId}/job/${test.lastRealFailureJobId}`);
+      }
+
+      // Show error for real failures
+      if (test.lastRealFailureError) {
+        const errorPreview = test.lastRealFailureError.length > 100
+          ? test.lastRealFailureError.substring(0, 100) + '...'
+          : test.lastRealFailureError;
+        console.log(`    💥 Error: ${errorPreview.replace(/\n/g, ' ')}`);
+      }
+    } else {
+      // Flaky tests (failed initially but eventually passed)
+      const retryPlural = test.totalRetries > 1 ? 'retries' : 'retry';
+      console.log(`    🟡 Flaky: eventually passed (${test.totalRetries} ${retryPlural})`);
+
+      // Show logs from when it failed (before retry succeeded)
+      if (test.flakyFailureJobId && test.flakyFailureRunId) {
+        console.log(`    🔗 Logs: https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${test.flakyFailureRunId}/job/${test.flakyFailureJobId}`);
+      }
+
+      // Show error from initial failure
+      if (test.flakyFailureError) {
+        const errorPreview = test.flakyFailureError.length > 100
+          ? test.flakyFailureError.substring(0, 100) + '...'
+          : test.flakyFailureError;
+        console.log(`    💥 Initial error: ${errorPreview.replace(/\n/g, ' ')}`);
+      }
+    }
+
+    console.log('');
+  }
+
+  if (summary.length > 10) {
+    console.log(`... and ${summary.length - 10} other tests\n`);
+  }
+}
+
+async function main() {
+  const github = new Octokit({ auth: env.GITHUB_TOKEN });
+
+  console.log('🧪🧐 Flaky Test Report\n');
+
+  const dateRange = getDateRange();
+  console.log(`Time range: ${dateRange.from} to ${dateRange.to}\n`);
+
+  try {
+    console.log('Fetching failed workflow runs...');
+    const failedRuns = await getFailedWorkflowRuns(github, dateRange.from, dateRange.to);
+
+    if (failedRuns.length === 0) {
+      console.log('✅ No failed tests found, great job!');
+      return;
+    }
+
+    console.log(`Found ${failedRuns.length} failed workflow run(s)`);
+    console.log('Downloading their test artifacts...');
+    const testData = await downloadTestArtifacts(github, failedRuns);
+
+    if (testData.length === 0) {
+      console.log('⚠️  No test artifacts found in failed runs');
+      return;
+    }
+
+    console.log('Analyzing test failures...');
+
+    // Two-pass approach: process real failures and flaky tests separately
+    const realFailures = extractRealFailures(testData);
+    const flakyTests = extractFlakyTests(testData);
+
+    const summary = summarizeFailures(realFailures, flakyTests);
+    displayResults(summary, dateRange.display);
+
+  } catch (error) {
+    console.error('❌ Error:', error.message);
+    if (error.status === 401) {
+      console.log('\n💡 This might be a GitHub token issue. Make sure your token has the right permissions.');
+    }
+    process.exit(1);
+  }
+}
+
+main().catch(error => {
+  console.error('\n❌ Unexpected error:', error);
+  process.exit(1);
+});

From b254e003203494ba6a0927f0ffcfaad146adfb1b Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Sat, 30 Aug 2025 20:28:23 +0200
Subject: [PATCH 02/13] ci: add flaky tests bot workflow

---
 .github/scripts/create-flaky-test-report.mjs  |  21 +--
 .../scripts/post-merge-validation-tracker.mjs |   2 +-
 package.json                                  |   4 +-
 yarn.lock                                     | 129 +++++++++++++++++-
 4 files changed, 142 insertions(+), 14 deletions(-)

diff --git a/.github/scripts/create-flaky-test-report.mjs b/.github/scripts/create-flaky-test-report.mjs
index 84b82509..11012821 100644
--- a/.github/scripts/create-flaky-test-report.mjs
+++ b/.github/scripts/create-flaky-test-report.mjs
@@ -1,24 +1,27 @@
 #!/usr/bin/env node
 
-import { Octokit } from 'octokit';
+import { Octokit } from '@octokit/rest';
 import unzipper from 'unzipper';
 
 const githubToken = process.env.GITHUB_TOKEN;
 if (!githubToken) throw new Error('Missing GITHUB_TOKEN env var');
 
 const env = {
-  OWNER: process.env.OWNER || 'MetaMask',
-  REPOSITORY: process.env.REPOSITORY || 'metamask-extension',
-  WORKFLOW_ID: process.env.WORKFLOW_ID || 'main.yml',
-  BRANCH: process.env.BRANCH || 'main',
   GITHUB_TOKEN: process.env.GITHUB_TOKEN,
   LOOKBACK_DAYS: parseInt(process.env.LOOKBACK_DAYS ?? '1'),
+  TEST_RESULTS_FILE_PATTERN: process.env.TEST_RESULTS_FILE_PATTERN || 'test-runs',
+  OWNER: process.env.OWNER || 'MetaMask',
+  REPOSITORY: process.env.REPOSITORY || 'metamask-mobile',
+  WORKFLOW_ID: process.env.WORKFLOW_ID || 'ci.yml',
+  BRANCH: process.env.BRANCH || 'main',
+  // For extension
+  // TEST_REPORT_ARTIFACTS: process.env.TEST_REPORT_ARTIFACTS
+  //   ? process.env.TEST_REPORT_ARTIFACTS.split(',').map(name => name.trim())
+  //   : ['test-e2e-chrome-report', 'test-e2e-firefox-report'],
+  // For mobile
   TEST_REPORT_ARTIFACTS: process.env.TEST_REPORT_ARTIFACTS
     ? process.env.TEST_REPORT_ARTIFACTS.split(',').map(name => name.trim())
-    : ['test-e2e-chrome-report', 'test-e2e-firefox-report'],
-  TEST_RESULTS_FILE_PATTERN: process.env.TEST_RESULTS_FILE_PATTERN || 'test-runs',
-  // For mobile, you can override these:
-  // REPOSITORY=metamask-mobile TEST_REPORT_ARTIFACTS=test-e2e-android-report,test-e2e-ios-report
+    : ['test-e2e-android-report', 'test-e2e-ios-report'],
 };
 
 function getDateRange() {
diff --git a/.github/scripts/post-merge-validation-tracker.mjs b/.github/scripts/post-merge-validation-tracker.mjs
index 68155ef9..8297ee0d 100644
--- a/.github/scripts/post-merge-validation-tracker.mjs
+++ b/.github/scripts/post-merge-validation-tracker.mjs
@@ -5,7 +5,7 @@ const githubToken = process.env.GITHUB_TOKEN;
 const spreadsheetId = process.env.SHEET_ID;
 const googleApplicationCredentialsBase64 = process.env.GOOGLE_APPLICATION_CREDENTIALS_BASE64;
 const repo = process.env.REPO;
-const LOOKBACK_DAYS = parseInt(process.env.LOOKBACK_DAYS ?? '2');
+const LOOKBACK_DAYS = parseInt(process.env.LOOKBACK_DAYS ?? '1');
 const START_HOUR_UTC = parseInt(process.env.START_HOUR_UTC ?? '7');
 
 const START_MINUTE_UTC = 0;
diff --git a/package.json b/package.json
index 9285b795..e55d22f1 100644
--- a/package.json
+++ b/package.json
@@ -35,7 +35,8 @@
     "luxon": "^3.3.0",
     "ora": "^5.4.1",
     "semver": "^7.7.2",
-    "simple-git": "3.27.0"
+    "simple-git": "3.27.0",
+    "unzipper": "^0.12.3"
   },
   "devDependencies": {
     "@lavamoat/allow-scripts": "^2.3.1",
@@ -49,6 +50,7 @@
     "@types/jest": "^28.1.6",
     "@types/node": "^20.3.2",
     "@types/semver": "^7",
+    "@types/unzipper": "^0",
     "@typescript-eslint/eslint-plugin": "^5.43.0",
     "@typescript-eslint/parser": "^5.43.0",
     "depcheck": "^1.4.3",
diff --git a/yarn.lock b/yarn.lock
index 7e92bef3..f55ae0fd 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1028,6 +1028,7 @@ __metadata:
     "@types/luxon": "npm:^3.3.0"
     "@types/node": "npm:^20.3.2"
     "@types/semver": "npm:^7"
+    "@types/unzipper": "npm:^0"
     "@typescript-eslint/eslint-plugin": "npm:^5.43.0"
     "@typescript-eslint/parser": "npm:^5.43.0"
     axios: "npm:^0.24.0"
@@ -1054,6 +1055,7 @@ __metadata:
     ts-jest: "npm:^28.0.7"
     ts-node: "npm:^10.9.1"
     typescript: "npm:^5.1.3"
+    unzipper: "npm:^0.12.3"
   languageName: unknown
   linkType: soft
 
@@ -1935,6 +1937,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/unzipper@npm:^0":
+  version: 0.10.11
+  resolution: "@types/unzipper@npm:0.10.11"
+  dependencies:
+    "@types/node": "npm:*"
+  checksum: 10/c11c0e072556038730b218ccf8af849911ed8a1338e6db863bdf4c44d53d83dd23e3de4752322b1e19cf0205ed6eaf8746e25aa3c2b38e419da457f9d6be7b4e
+  languageName: node
+  linkType: hard
+
 "@types/yargs-parser@npm:*":
   version: 15.0.0
   resolution: "@types/yargs-parser@npm:15.0.0"
@@ -2624,6 +2635,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"bluebird@npm:~3.7.2":
+  version: 3.7.2
+  resolution: "bluebird@npm:3.7.2"
+  checksum: 10/007c7bad22c5d799c8dd49c85b47d012a1fe3045be57447721e6afbd1d5be43237af1db62e26cb9b0d9ba812d2e4ca3bac82f6d7e016b6b88de06ee25ceb96e7
+  languageName: node
+  linkType: hard
+
 "brace-expansion@npm:^1.1.7":
   version: 1.1.11
   resolution: "brace-expansion@npm:1.1.11"
@@ -3085,6 +3103,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"core-util-is@npm:~1.0.0":
+  version: 1.0.3
+  resolution: "core-util-is@npm:1.0.3"
+  checksum: 10/9de8597363a8e9b9952491ebe18167e3b36e7707569eed0ebf14f8bba773611376466ae34575bca8cfe3c767890c859c74056084738f09d4e4a6f902b2ad7d99
+  languageName: node
+  linkType: hard
+
 "cosmiconfig@npm:^7.0.0":
   version: 7.1.0
   resolution: "cosmiconfig@npm:7.1.0"
@@ -3372,6 +3397,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"duplexer2@npm:~0.1.4":
+  version: 0.1.4
+  resolution: "duplexer2@npm:0.1.4"
+  dependencies:
+    readable-stream: "npm:^2.0.2"
+  checksum: 10/f60ff8b8955f992fd9524516e82faa5662d7aca5b99ee71c50bbbe1a3c970fafacb35d526d8b05cef8c08be56eed3663c096c50626c3c3651a52af36c408bf4d
+  languageName: node
+  linkType: hard
+
 "ecdsa-sig-formatter@npm:1.0.11, ecdsa-sig-formatter@npm:^1.0.11":
   version: 1.0.11
   resolution: "ecdsa-sig-formatter@npm:1.0.11"
@@ -4240,6 +4274,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"fs-extra@npm:^11.2.0":
+  version: 11.3.1
+  resolution: "fs-extra@npm:11.3.1"
+  dependencies:
+    graceful-fs: "npm:^4.2.0"
+    jsonfile: "npm:^6.0.1"
+    universalify: "npm:^2.0.0"
+  checksum: 10/2b893213411b1da11f9b061ccb0bcff4d6dd66fe90aa8f5b1616219a5e7ca659da869f454ebd8e94aa21c58342730fb43a2e5c98b5c6c5124f0c54a4633f64b0
+  languageName: node
+  linkType: hard
+
 "fs-minipass@npm:^2.0.0, fs-minipass@npm:^2.1.0":
   version: 2.1.0
   resolution: "fs-minipass@npm:2.1.0"
@@ -4611,6 +4656,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.2":
+  version: 4.2.11
+  resolution: "graceful-fs@npm:4.2.11"
+  checksum: 10/bf152d0ed1dc159239db1ba1f74fdbc40cb02f626770dcd5815c427ce0688c2635a06ed69af364396da4636d0408fcf7d4afdf7881724c3307e46aff30ca49e2
+  languageName: node
+  linkType: hard
+
 "graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9":
   version: 4.2.10
   resolution: "graceful-fs@npm:4.2.10"
@@ -4874,7 +4926,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"inherits@npm:2, inherits@npm:^2.0.3, inherits@npm:^2.0.4":
+"inherits@npm:2, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.3":
   version: 2.0.4
   resolution: "inherits@npm:2.0.4"
   checksum: 10/cd45e923bee15186c07fa4c89db0aace24824c482fb887b528304694b2aa6ff8a898da8657046a5dcf3e46cd6db6c61629551f9215f208d7c3f157cf9b290521
@@ -5183,6 +5235,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"isarray@npm:~1.0.0":
+  version: 1.0.0
+  resolution: "isarray@npm:1.0.0"
+  checksum: 10/f032df8e02dce8ec565cf2eb605ea939bdccea528dbcf565cdf92bfa2da9110461159d86a537388ef1acef8815a330642d7885b29010e8f7eac967c9993b65ab
+  languageName: node
+  linkType: hard
+
 "isexe@npm:^2.0.0":
   version: 2.0.0
   resolution: "isexe@npm:2.0.0"
@@ -5811,6 +5870,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"jsonfile@npm:^6.0.1":
+  version: 6.2.0
+  resolution: "jsonfile@npm:6.2.0"
+  dependencies:
+    graceful-fs: "npm:^4.1.6"
+    universalify: "npm:^2.0.0"
+  dependenciesMeta:
+    graceful-fs:
+      optional: true
+  checksum: 10/513aac94a6eff070767cafc8eb4424b35d523eec0fcd8019fe5b975f4de5b10a54640c8d5961491ddd8e6f562588cf62435c5ddaf83aaf0986cd2ee789e0d7b9
+  languageName: node
+  linkType: hard
+
 "jwa@npm:^2.0.0":
   version: 2.0.0
   resolution: "jwa@npm:2.0.0"
@@ -6836,6 +6908,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"process-nextick-args@npm:~2.0.0":
+  version: 2.0.1
+  resolution: "process-nextick-args@npm:2.0.1"
+  checksum: 10/1d38588e520dab7cea67cbbe2efdd86a10cc7a074c09657635e34f035277b59fbb57d09d8638346bf7090f8e8ebc070c96fa5fd183b777fff4f5edff5e9466cf
+  languageName: node
+  linkType: hard
+
 "promise-inflight@npm:^1.0.1":
   version: 1.0.1
   resolution: "promise-inflight@npm:1.0.1"
@@ -6951,6 +7030,21 @@ __metadata:
   languageName: node
   linkType: hard
 
+"readable-stream@npm:^2.0.2":
+  version: 2.3.8
+  resolution: "readable-stream@npm:2.3.8"
+  dependencies:
+    core-util-is: "npm:~1.0.0"
+    inherits: "npm:~2.0.3"
+    isarray: "npm:~1.0.0"
+    process-nextick-args: "npm:~2.0.0"
+    safe-buffer: "npm:~5.1.1"
+    string_decoder: "npm:~1.1.1"
+    util-deprecate: "npm:~1.0.1"
+  checksum: 10/8500dd3a90e391d6c5d889256d50ec6026c059fadee98ae9aa9b86757d60ac46fff24fafb7a39fa41d54cb39d8be56cc77be202ebd4cd8ffcf4cb226cbaa40d4
+  languageName: node
+  linkType: hard
+
 "readable-stream@npm:^3.4.0, readable-stream@npm:^3.6.0":
   version: 3.6.2
   resolution: "readable-stream@npm:3.6.2"
@@ -7161,7 +7255,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"safe-buffer@npm:~5.1.1":
+"safe-buffer@npm:~5.1.0, safe-buffer@npm:~5.1.1":
   version: 5.1.2
   resolution: "safe-buffer@npm:5.1.2"
   checksum: 10/7eb5b48f2ed9a594a4795677d5a150faa7eb54483b2318b568dc0c4fc94092a6cce5be02c7288a0500a156282f5276d5688bce7259299568d1053b2150ef374a
@@ -7616,6 +7710,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"string_decoder@npm:~1.1.1":
+  version: 1.1.1
+  resolution: "string_decoder@npm:1.1.1"
+  dependencies:
+    safe-buffer: "npm:~5.1.0"
+  checksum: 10/7c41c17ed4dea105231f6df208002ebddd732e8e9e2d619d133cecd8e0087ddfd9587d2feb3c8caf3213cbd841ada6d057f5142cae68a4e62d3540778d9819b4
+  languageName: node
+  linkType: hard
+
 "strip-ansi@npm:^6.0.0, strip-ansi@npm:^6.0.1":
   version: 6.0.1
   resolution: "strip-ansi@npm:6.0.1"
@@ -8075,6 +8178,26 @@ __metadata:
   languageName: node
   linkType: hard
 
+"universalify@npm:^2.0.0":
+  version: 2.0.1
+  resolution: "universalify@npm:2.0.1"
+  checksum: 10/ecd8469fe0db28e7de9e5289d32bd1b6ba8f7183db34f3bfc4ca53c49891c2d6aa05f3fb3936a81285a905cc509fb641a0c3fc131ec786167eff41236ae32e60
+  languageName: node
+  linkType: hard
+
+"unzipper@npm:^0.12.3":
+  version: 0.12.3
+  resolution: "unzipper@npm:0.12.3"
+  dependencies:
+    bluebird: "npm:~3.7.2"
+    duplexer2: "npm:~0.1.4"
+    fs-extra: "npm:^11.2.0"
+    graceful-fs: "npm:^4.2.2"
+    node-int64: "npm:^0.4.0"
+  checksum: 10/b210c421308e1913e01b54faad4ae79e758c674311892414a0697acacba9f82fa0051b677faa77e62fab422eef928c858f2d5cda9ddb47a2f3db95b0e9b36359
+  languageName: node
+  linkType: hard
+
 "update-browserslist-db@npm:^1.0.5":
   version: 1.0.5
   resolution: "update-browserslist-db@npm:1.0.5"
@@ -8105,7 +8228,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"util-deprecate@npm:^1.0.1":
+"util-deprecate@npm:^1.0.1, util-deprecate@npm:~1.0.1":
   version: 1.0.2
   resolution: "util-deprecate@npm:1.0.2"
   checksum: 10/474acf1146cb2701fe3b074892217553dfcf9a031280919ba1b8d651a068c9b15d863b7303cb15bd00a862b498e6cf4ad7b4a08fb134edd5a6f7641681cb54a2

From 8a41fb50fad41ca4f39503fda3e4624c3b138b6c Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Sat, 30 Aug 2025 20:30:35 +0200
Subject: [PATCH 03/13] ci: add flaky tests bot workflow

---
 yarn.lock | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index f55ae0fd..3dcf50be 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4656,20 +4656,13 @@ __metadata:
   languageName: node
   linkType: hard
 
-"graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.2":
+"graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.2, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9":
   version: 4.2.11
   resolution: "graceful-fs@npm:4.2.11"
   checksum: 10/bf152d0ed1dc159239db1ba1f74fdbc40cb02f626770dcd5815c427ce0688c2635a06ed69af364396da4636d0408fcf7d4afdf7881724c3307e46aff30ca49e2
   languageName: node
   linkType: hard
 
-"graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9":
-  version: 4.2.10
-  resolution: "graceful-fs@npm:4.2.10"
-  checksum: 10/0c83c52b62c68a944dcfb9d66b0f9f10f7d6e3d081e8067b9bfdc9e5f3a8896584d576036f82915773189eec1eba599397fc620e75c03c0610fb3d67c6713c1a
-  languageName: node
-  linkType: hard
-
 "graphemer@npm:^1.4.0":
   version: 1.4.0
   resolution: "graphemer@npm:1.4.0"

From 702372352d5dafc19a0278512556ed769272237a Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Tue, 2 Sep 2025 18:49:36 +0200
Subject: [PATCH 04/13] ci: add flaky tests bot workflow

---
 .github/scripts/create-flaky-test-report.mjs | 246 +++++++++++++++++--
 package.json                                 |   1 +
 yarn.lock                                    |  71 ++++++
 3 files changed, 298 insertions(+), 20 deletions(-)

diff --git a/.github/scripts/create-flaky-test-report.mjs b/.github/scripts/create-flaky-test-report.mjs
index 11012821..215b4a9e 100644
--- a/.github/scripts/create-flaky-test-report.mjs
+++ b/.github/scripts/create-flaky-test-report.mjs
@@ -2,6 +2,7 @@
 
 import { Octokit } from '@octokit/rest';
 import unzipper from 'unzipper';
+import { IncomingWebhook } from '@slack/webhook';
 
 const githubToken = process.env.GITHUB_TOKEN;
 if (!githubToken) throw new Error('Missing GITHUB_TOKEN env var');
@@ -14,14 +15,10 @@ const env = {
   REPOSITORY: process.env.REPOSITORY || 'metamask-mobile',
   WORKFLOW_ID: process.env.WORKFLOW_ID || 'ci.yml',
   BRANCH: process.env.BRANCH || 'main',
-  // For extension
-  // TEST_REPORT_ARTIFACTS: process.env.TEST_REPORT_ARTIFACTS
-  //   ? process.env.TEST_REPORT_ARTIFACTS.split(',').map(name => name.trim())
-  //   : ['test-e2e-chrome-report', 'test-e2e-firefox-report'],
-  // For mobile
+  SLACK_WEBHOOK_FLAKY_TESTS: process.env.SLACK_WEBHOOK_FLAKY_TESTS || '',
   TEST_REPORT_ARTIFACTS: process.env.TEST_REPORT_ARTIFACTS
     ? process.env.TEST_REPORT_ARTIFACTS.split(',').map(name => name.trim())
-    : ['test-e2e-android-report', 'test-e2e-ios-report'],
+    : ['test-e2e-android-report', 'test-e2e-ios-report', 'test-e2e-chrome-report', 'test-e2e-firefox-report'],
 };
 
 function getDateRange() {
@@ -29,7 +26,6 @@ function getDateRange() {
   const daysAgo = new Date(today.getTime() - (env.LOOKBACK_DAYS * 24 * 60 * 60 * 1000));
 
   const fromDisplay = daysAgo.toLocaleDateString('en-US', {
-    weekday: 'short',
     month: 'short',
     day: 'numeric',
     hour: 'numeric',
@@ -37,7 +33,6 @@ function getDateRange() {
   });
 
   const toDisplay = today.toLocaleDateString('en-US', {
-    weekday: 'short',
     month: 'short',
     day: 'numeric',
     hour: 'numeric',
@@ -51,7 +46,7 @@ function getDateRange() {
   };
 }
 
-async function getFailedWorkflowRuns(github, from, to) {
+async function getWorkflowRuns(github, from, to) {
   try {
     const runs = await github.paginate(
       github.rest.actions.listWorkflowRuns,
@@ -60,7 +55,6 @@ async function getFailedWorkflowRuns(github, from, to) {
         repo: env.REPOSITORY,
         workflow_id: env.WORKFLOW_ID,
         branch: env.BRANCH,
-        status: 'failure',
         created: `${from}..${to}`,
         per_page: 100,
       }
@@ -320,9 +314,218 @@ function summarizeFailures(realFailures, flakyTests = []) {
     });
 }
 
+async function sendSlackReport(summary, dateDisplay, workflowCount) {
+  if (!env.SLACK_WEBHOOK_FLAKY_TESTS || !env.SLACK_WEBHOOK_FLAKY_TESTS.startsWith('https://')) {
+    console.log('Skipping Slack notification');
+    return;
+  }
+
+  console.log('\n📤 Sending report to Slack...');
+  try {
+    const webhook = new IncomingWebhook(env.SLACK_WEBHOOK_FLAKY_TESTS);
+    const blocks = createSlackBlocks(summary, dateDisplay, workflowCount);
+
+    // Slack has a limit of 50 blocks per message
+    const BATCH_SIZE = 50;
+    for (let i = 0; i < blocks.length; i += BATCH_SIZE) {
+      const batch = blocks.slice(i, i + BATCH_SIZE);
+      await webhook.send({ blocks: batch });
+    }
+
+    console.log('✅ Report sent to Slack successfully');
+  } catch (slackError) {
+    console.error('❌ Failed to send Slack notification:', slackError.message);
+  }
+}
+
+function createSlackBlocks(summary, dateDisplay, workflowCount = 0) {
+  const blocks = [];
+
+  blocks.push({
+    type: 'header',
+    text: {
+      type: 'plain_text',
+      text: 'Flaky Test Report',
+      emoji: true
+    }
+  });
+
+  // Calculate counts first
+  const realFailures = summary.filter(test => test.realFailures > 0);
+  const flakyTests = summary.filter(test => test.realFailures === 0);
+
+  blocks.push({
+    type: 'context',
+    elements: [{
+      type: 'mrkdwn',
+      text: `Period (UTC): ${dateDisplay} | Repo: ${env.REPOSITORY} | Branch: ${env.BRANCH} | ${workflowCount} CI runs | Found: ${realFailures.length} failures, ${flakyTests.length} flaky`
+    }]
+  });
+
+  blocks.push({ type: 'divider' });
+
+  if (summary.length === 0) {
+    blocks.push({
+      type: 'rich_text',
+      elements: [{
+        type: 'rich_text_section',
+        elements: [
+          { type: 'text', text: 'No flaky tests found, great job! ✅ ' }
+        ]
+      }]
+    });
+    return blocks;
+  }
+
+  const top10 = summary.slice(0, 10);
+
+  // Real failures section
+  if (realFailures.length > 0) {
+    blocks.push({
+      type: 'rich_text',
+      elements: [{
+        type: 'rich_text_section',
+        elements: [
+          { type: 'emoji', name: 'x' },
+          { type: 'text', text: ' ' },
+          { type: 'text', text: 'Failures', style: { bold: true } }
+        ]
+      }]
+    });
+
+    // Each failure
+    top10.filter(test => test.realFailures > 0).forEach((test, idx) => {
+      const globalIndex = top10.indexOf(test) + 1;
+      const failText = test.realFailures === 1 ? 'time' : 'times';
+      const retryText = test.totalRetries === 1 ? 'retry' : 'retries';
+
+      // Create GitHub file URL
+      const fileUrl = `https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${test.path}`;
+
+      // Build elements for this test
+      const elements = [
+        { type: 'text', text: `  ${globalIndex}. ` },  // 2 spaces indent
+        { type: 'link', url: fileUrl, text: test.name },
+        { type: 'text', text: ` (failed ${test.realFailures} ${failText}, ${test.totalRetries} ${retryText})`, style: { bold: true } }
+      ];
+
+      if (test.lastRealFailureJobId && test.lastRealFailureRunId) {
+        const jobUrl = `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${test.lastRealFailureRunId}/job/${test.lastRealFailureJobId}`;
+        elements.push(
+          { type: 'text', text: ' - ' },
+          { type: 'link', url: jobUrl, text: 'last log' }
+        );
+      }
+
+      blocks.push({
+        type: 'rich_text',
+        elements: [{
+          type: 'rich_text_section',
+          elements: elements
+        }]
+      });
+
+      // Error message (if exists)
+      const error = test.lastRealFailureError;
+      if (error) {
+        const errorPreview = error.length > 150 ? error.substring(0, 150) + '...' : error;
+        blocks.push({
+          type: 'rich_text',
+          elements: [{
+            type: 'rich_text_section',
+            elements: [
+              { type: 'text', text: `     ${errorPreview.replace(/\n/g, ' ')}` }  // 5 spaces indent for error
+            ]
+          }]
+        });
+      }
+    });
+  }
+
+  // Divider between sections if both exist
+  if (realFailures.length > 0 && flakyTests.length > 0) {
+    blocks.push({ type: 'divider' });
+  }
+
+  // Flaky tests section
+  if (flakyTests.length > 0) {
+    // Title
+    blocks.push({
+      type: 'rich_text',
+      elements: [{
+        type: 'rich_text_section',
+        elements: [
+          { type: 'emoji', name: 'large_yellow_circle' },
+          { type: 'text', text: ' ' },
+          { type: 'text', text: 'Flaky (eventually passed)', style: { bold: true } }
+        ]
+      }]
+    });
+
+    // Each flaky test
+    top10.filter(test => test.realFailures === 0).forEach((test, idx) => {
+      const globalIndex = top10.indexOf(test) + 1;
+      const retryText = test.totalRetries === 1 ? 'retry' : 'retries';
+
+      // Create GitHub file URL
+      const fileUrl = `https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${test.path}`;
+
+      // Build elements for this test
+      const elements = [
+        { type: 'text', text: `  ${globalIndex}. ` },  // 2 spaces indent
+        { type: 'link', url: fileUrl, text: test.name },
+        { type: 'text', text: ` (${test.totalRetries} ${retryText})`, style: { bold: true } }
+      ];
+
+      if (test.flakyFailureJobId && test.flakyFailureRunId) {
+        const jobUrl = `https://github.com/${env.OWNER}/${env.REPOSITORY}/actions/runs/${test.flakyFailureRunId}/job/${test.flakyFailureJobId}`;
+        elements.push(
+          { type: 'text', text: ' - ' },
+          { type: 'link', url: jobUrl, text: 'last log' }
+        );
+      }
+
+      blocks.push({
+        type: 'rich_text',
+        elements: [{
+          type: 'rich_text_section',
+          elements: elements
+        }]
+      });
+
+      // Error message (if exists)
+      const error = test.flakyFailureError;
+      if (error) {
+        const errorPreview = error.length > 150 ? error.substring(0, 150) + '...' : error;
+        blocks.push({
+          type: 'rich_text',
+          elements: [{
+            type: 'rich_text_section',
+            elements: [
+              { type: 'text', text: `     ${errorPreview.replace(/\n/g, ' ')}` }  // 5 spaces indent for error
+            ]
+          }]
+        });
+      }
+    });
+  }
+
+  if (summary.length > 10) {
+    blocks.push({
+      type: 'context',
+      elements: [{
+        type: 'mrkdwn',
+        text: `_... and ${summary.length - 10} other tests_`
+      }]
+    });
+  }
+
+  return blocks;
+}
+
 function displayResults(summary, dateDisplay) {
   console.log('\n' + '='.repeat(80));
-  console.log(`📊 FLAKY TEST REPORT - ${dateDisplay}`);
+  console.log(`📊 REPORT - ${dateDisplay}`);
   console.log('='.repeat(80));
 
   if (summary.length === 0) {
@@ -334,7 +537,9 @@ function displayResults(summary, dateDisplay) {
   const flakyTests = summary.filter(test => test.realFailures === 0);
 
   console.log(`${realFailures.length} real failures (failed even after retries)`);
-  console.log(`${flakyTests.length} flaky tests (eventually passed after retries)\n`);
+  console.log(`${flakyTests.length} flaky tests (eventually passed after retries)`);
+  console.log(`\n📌 Sorted by: 1) Number of failures ↓  2) Total retries ↓`);
+  console.log(`📊 Numbers shown are cumulative across all runs in the time period\n`);
 
   const top10 = summary.slice(0, 10);
 
@@ -346,7 +551,7 @@ function displayResults(summary, dateDisplay) {
       // Real failures (tests that failed even after retries)
       const failurePlural = test.realFailures > 1 ? 's' : '';
       const retryPlural = test.totalRetries > 1 ? 'retries' : 'retry';
-      const retryText = test.totalRetries > 0 ? ` (${test.totalRetries} ${retryPlural})` : '';
+      const retryText = test.totalRetries > 0 ? ` (${test.totalRetries} total ${retryPlural})` : '';
       console.log(`    ❌ Failed: ${test.realFailures} time${failurePlural}${retryText}`);
 
       // Show logs for real failures
@@ -364,7 +569,7 @@ function displayResults(summary, dateDisplay) {
     } else {
       // Flaky tests (failed initially but eventually passed)
       const retryPlural = test.totalRetries > 1 ? 'retries' : 'retry';
-      console.log(`    🟡 Flaky: eventually passed (${test.totalRetries} ${retryPlural})`);
+      console.log(`    🟡 Flaky: eventually passed (${test.totalRetries} total ${retryPlural})`);
 
       // Show logs from when it failed (before retry succeeded)
       if (test.flakyFailureJobId && test.flakyFailureRunId) {
@@ -397,17 +602,17 @@ async function main() {
   console.log(`Time range: ${dateRange.from} to ${dateRange.to}\n`);
 
   try {
-    console.log('Fetching failed workflow runs...');
-    const failedRuns = await getFailedWorkflowRuns(github, dateRange.from, dateRange.to);
+    console.log('Fetching workflow runs...');
+    const workflowRuns = await getWorkflowRuns(github, dateRange.from, dateRange.to);
 
-    if (failedRuns.length === 0) {
-      console.log('✅ No failed tests found, great job!');
+    if (workflowRuns.length === 0) {
+      console.log('⚠️ No workflow runs found.');
       return;
     }
 
-    console.log(`Found ${failedRuns.length} failed workflow run(s)`);
+    console.log(`Found ${workflowRuns.length} workflow run(s)`);
     console.log('Downloading their test artifacts...');
-    const testData = await downloadTestArtifacts(github, failedRuns);
+    const testData = await downloadTestArtifacts(github, workflowRuns);
 
     if (testData.length === 0) {
       console.log('⚠️  No test artifacts found in failed runs');
@@ -422,6 +627,7 @@ async function main() {
 
     const summary = summarizeFailures(realFailures, flakyTests);
     displayResults(summary, dateRange.display);
+    await sendSlackReport(summary, dateRange.display, workflowRuns.length);
 
   } catch (error) {
     console.error('❌ Error:', error.message);
diff --git a/package.json b/package.json
index e55d22f1..4eb16b0d 100644
--- a/package.json
+++ b/package.json
@@ -27,6 +27,7 @@
     "@octokit/request": "^8.1.1",
     "@octokit/rest": "^19.0.13",
     "@slack/web-api": "^6.0.0",
+    "@slack/webhook": "^7.0.6",
     "@types/luxon": "^3.3.0",
     "axios": "^0.24.0",
     "csv-parse": "5.6.0",
diff --git a/yarn.lock b/yarn.lock
index 3dcf50be..f3302485 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1022,6 +1022,7 @@ __metadata:
     "@octokit/request": "npm:^8.1.1"
     "@octokit/rest": "npm:^19.0.13"
     "@slack/web-api": "npm:^6.0.0"
+    "@slack/webhook": "npm:^7.0.6"
     "@swc/cli": "npm:^0.1.62"
     "@swc/core": "npm:^1.3.80"
     "@types/jest": "npm:^28.1.6"
@@ -1478,6 +1479,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@slack/types@npm:^2.9.0":
+  version: 2.16.0
+  resolution: "@slack/types@npm:2.16.0"
+  checksum: 10/e18b568a47d94e9e7234dfd06f789224d6804edae4a2f31068b3f388ce4c482a6dbc6c035dc3dec63e5723f211f92c7694ee40b2ec83d4ac90d46bb35fa46eb5
+  languageName: node
+  linkType: hard
+
 "@slack/web-api@npm:^6.0.0":
   version: 6.13.0
   resolution: "@slack/web-api@npm:6.13.0"
@@ -1497,6 +1505,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@slack/webhook@npm:^7.0.6":
+  version: 7.0.6
+  resolution: "@slack/webhook@npm:7.0.6"
+  dependencies:
+    "@slack/types": "npm:^2.9.0"
+    "@types/node": "npm:>=18.0.0"
+    axios: "npm:^1.11.0"
+  checksum: 10/8f8083f9654e590f04731985b337f576842b2034a9261010f85d813c4e262f69d856c142b0dcf2022bfe69c22c2e97cc7d877a79989cd0f7a0cf2554ae0754ed
+  languageName: node
+  linkType: hard
+
 "@swc/cli@npm:^0.1.62":
   version: 0.1.62
   resolution: "@swc/cli@npm:0.1.62"
@@ -1886,6 +1905,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/node@npm:>=18.0.0":
+  version: 24.3.0
+  resolution: "@types/node@npm:24.3.0"
+  dependencies:
+    undici-types: "npm:~7.10.0"
+  checksum: 10/1331c2d0e9a512ac27a016b4df3eff92317e4603dbbbab31731275dff14d3a04847a50c5776cbf94f99ff4dedac0ba5f721dce8cea020d8eea5e21711fd964b0
+  languageName: node
+  linkType: hard
+
 "@types/node@npm:^20.3.2":
   version: 20.3.2
   resolution: "@types/node@npm:20.3.2"
@@ -2459,6 +2487,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"axios@npm:^1.11.0":
+  version: 1.11.0
+  resolution: "axios@npm:1.11.0"
+  dependencies:
+    follow-redirects: "npm:^1.15.6"
+    form-data: "npm:^4.0.4"
+    proxy-from-env: "npm:^1.1.0"
+  checksum: 10/232df4af7a4e4e07baa84621b9cc4b0c518a757b4eacc7f635c0eb3642cb98dff347326739f24b891b3b4481b7b838c79a3a0c4819c9fbc1fc40232431b9c5dc
+  languageName: node
+  linkType: hard
+
 "axios@npm:^1.7.4":
   version: 1.7.9
   resolution: "axios@npm:1.7.9"
@@ -3558,6 +3597,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"es-set-tostringtag@npm:^2.1.0":
+  version: 2.1.0
+  resolution: "es-set-tostringtag@npm:2.1.0"
+  dependencies:
+    es-errors: "npm:^1.3.0"
+    get-intrinsic: "npm:^1.2.6"
+    has-tostringtag: "npm:^1.0.2"
+    hasown: "npm:^2.0.2"
+  checksum: 10/86814bf8afbcd8966653f731415888019d4bc4aca6b6c354132a7a75bb87566751e320369654a101d23a91c87a85c79b178bcf40332839bd347aff437c4fb65f
+  languageName: node
+  linkType: hard
+
 "es-shim-unscopables@npm:^1.0.0":
   version: 1.0.0
   resolution: "es-shim-unscopables@npm:1.0.0"
@@ -4274,6 +4325,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"form-data@npm:^4.0.4":
+  version: 4.0.4
+  resolution: "form-data@npm:4.0.4"
+  dependencies:
+    asynckit: "npm:^0.4.0"
+    combined-stream: "npm:^1.0.8"
+    es-set-tostringtag: "npm:^2.1.0"
+    hasown: "npm:^2.0.2"
+    mime-types: "npm:^2.1.12"
+  checksum: 10/a4b62e21932f48702bc468cc26fb276d186e6b07b557e3dd7cc455872bdbb82db7db066844a64ad3cf40eaf3a753c830538183570462d3649fdfd705601cbcfb
+  languageName: node
+  linkType: hard
+
 "fs-extra@npm:^11.2.0":
   version: 11.3.1
   resolution: "fs-extra@npm:11.3.1"
@@ -8139,6 +8203,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"undici-types@npm:~7.10.0":
+  version: 7.10.0
+  resolution: "undici-types@npm:7.10.0"
+  checksum: 10/1f3fe777937690ab8a7a7bccabc8fdf4b3171f4899b5a384fb5f3d6b56c4b5fec2a51fbf345c9dd002ff6716fd440a37fa8fdb0e13af8eca8889f25445875ba3
+  languageName: node
+  linkType: hard
+
 "unicorn-magic@npm:^0.3.0":
   version: 0.3.0
   resolution: "unicorn-magic@npm:0.3.0"

From 2e07638567a30b081561b87386b7bfe480ebfda3 Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Wed, 3 Sep 2025 13:53:45 +0200
Subject: [PATCH 05/13] ci: add flaky tests bot workflow

---
 .github/scripts/create-flaky-test-report.mjs | 55 ++++++++++++--------
 1 file changed, 32 insertions(+), 23 deletions(-)

diff --git a/.github/scripts/create-flaky-test-report.mjs b/.github/scripts/create-flaky-test-report.mjs
index 215b4a9e..14a24b9c 100644
--- a/.github/scripts/create-flaky-test-report.mjs
+++ b/.github/scripts/create-flaky-test-report.mjs
@@ -12,8 +12,8 @@ const env = {
   LOOKBACK_DAYS: parseInt(process.env.LOOKBACK_DAYS ?? '1'),
   TEST_RESULTS_FILE_PATTERN: process.env.TEST_RESULTS_FILE_PATTERN || 'test-runs',
   OWNER: process.env.OWNER || 'MetaMask',
-  REPOSITORY: process.env.REPOSITORY || 'metamask-mobile',
-  WORKFLOW_ID: process.env.WORKFLOW_ID || 'ci.yml',
+  REPOSITORY: process.env.REPOSITORY || 'metamask-extension',
+  WORKFLOW_ID: process.env.WORKFLOW_ID || 'main.yml',
   BRANCH: process.env.BRANCH || 'main',
   SLACK_WEBHOOK_FLAKY_TESTS: process.env.SLACK_WEBHOOK_FLAKY_TESTS || '',
   TEST_REPORT_ARTIFACTS: process.env.TEST_REPORT_ARTIFACTS
@@ -60,7 +60,13 @@ async function getWorkflowRuns(github, from, to) {
       }
     );
 
-    return runs;
+    // Filter to only completed runs
+    const completedRuns = runs.filter(run => run.status === 'completed');
+
+    // Sort by created date (newest first)
+    completedRuns.sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
+
+    return completedRuns;
   } catch (error) {
     if (error.status === 404) {
       throw new Error(`Workflow '${env.WORKFLOW_ID}' not found in ${env.OWNER}/${env.REPOSITORY}`);
@@ -314,7 +320,7 @@ function summarizeFailures(realFailures, flakyTests = []) {
     });
 }
 
-async function sendSlackReport(summary, dateDisplay, workflowCount) {
+async function sendSlackReport(summary, dateDisplay, workflowCount, failedCount) {
   if (!env.SLACK_WEBHOOK_FLAKY_TESTS || !env.SLACK_WEBHOOK_FLAKY_TESTS.startsWith('https://')) {
     console.log('Skipping Slack notification');
     return;
@@ -323,7 +329,7 @@ async function sendSlackReport(summary, dateDisplay, workflowCount) {
   console.log('\n📤 Sending report to Slack...');
   try {
     const webhook = new IncomingWebhook(env.SLACK_WEBHOOK_FLAKY_TESTS);
-    const blocks = createSlackBlocks(summary, dateDisplay, workflowCount);
+    const blocks = createSlackBlocks(summary, dateDisplay, workflowCount, failedCount);
 
     // Slack has a limit of 50 blocks per message
     const BATCH_SIZE = 50;
@@ -338,14 +344,14 @@ async function sendSlackReport(summary, dateDisplay, workflowCount) {
   }
 }
 
-function createSlackBlocks(summary, dateDisplay, workflowCount = 0) {
+function createSlackBlocks(summary, dateDisplay, workflowCount = 0, failedCount = 0) {
   const blocks = [];
 
   blocks.push({
     type: 'header',
     text: {
       type: 'plain_text',
-      text: 'Flaky Test Report',
+      text: 'Flaky Test Report - Top 10',
       emoji: true
     }
   });
@@ -358,7 +364,7 @@ function createSlackBlocks(summary, dateDisplay, workflowCount = 0) {
     type: 'context',
     elements: [{
       type: 'mrkdwn',
-      text: `Period (UTC): ${dateDisplay} | Repo: ${env.REPOSITORY} | Branch: ${env.BRANCH} | ${workflowCount} CI runs | Found: ${realFailures.length} failures, ${flakyTests.length} flaky`
+      text: `Period (UTC): ${dateDisplay} | Repo: ${env.REPOSITORY} | Failed CI Runs: ${failedCount}/${workflowCount} from ${env.BRANCH} branch\nFound: ${realFailures.length} tests failing, ${flakyTests.length} flaky (eventually passed)`
     }]
   });
 
@@ -434,7 +440,7 @@ function createSlackBlocks(summary, dateDisplay, workflowCount = 0) {
           elements: [{
             type: 'rich_text_section',
             elements: [
-              { type: 'text', text: `     ${errorPreview.replace(/\n/g, ' ')}` }  // 5 spaces indent for error
+              { type: 'text', text: `  ${errorPreview.replace(/\n/g, ' ')}`, style: { italic: true } }
             ]
           }]
         });
@@ -442,6 +448,10 @@ function createSlackBlocks(summary, dateDisplay, workflowCount = 0) {
     });
   }
 
+  if (realFailures.length >= 10) {
+    return blocks;
+  }
+
   // Divider between sections if both exist
   if (realFailures.length > 0 && flakyTests.length > 0) {
     blocks.push({ type: 'divider' });
@@ -462,9 +472,13 @@ function createSlackBlocks(summary, dateDisplay, workflowCount = 0) {
       }]
     });
 
-    // Each flaky test
-    top10.filter(test => test.realFailures === 0).forEach((test, idx) => {
-      const globalIndex = top10.indexOf(test) + 1;
+    // Each flaky test (respecting the 10-item limit)
+    const displayedRealFailures = Math.min(realFailures.length, 10);
+    const remainingSlots = 10 - displayedRealFailures;
+    const flakyTestsToShow = flakyTests.slice(0, remainingSlots);
+
+    flakyTestsToShow.forEach((test, idx) => {
+      const globalIndex = displayedRealFailures + idx + 1;
       const retryText = test.totalRetries === 1 ? 'retry' : 'retries';
 
       // Create GitHub file URL
@@ -510,16 +524,6 @@ function createSlackBlocks(summary, dateDisplay, workflowCount = 0) {
     });
   }
 
-  if (summary.length > 10) {
-    blocks.push({
-      type: 'context',
-      elements: [{
-        type: 'mrkdwn',
-        text: `_... and ${summary.length - 10} other tests_`
-      }]
-    });
-  }
-
   return blocks;
 }
 
@@ -611,6 +615,11 @@ async function main() {
     }
 
     console.log(`Found ${workflowRuns.length} workflow run(s)`);
+
+    // Count failed runs
+    const failedRuns = workflowRuns.filter(run => run.conclusion !== 'success');
+    console.log(`Failed CI Runs: ${failedRuns.length}/${workflowRuns.length} from ${env.BRANCH}`);
+
     console.log('Downloading their test artifacts...');
     const testData = await downloadTestArtifacts(github, workflowRuns);
 
@@ -627,7 +636,7 @@ async function main() {
 
     const summary = summarizeFailures(realFailures, flakyTests);
     displayResults(summary, dateRange.display);
-    await sendSlackReport(summary, dateRange.display, workflowRuns.length);
+    await sendSlackReport(summary, dateRange.display, workflowRuns.length, failedRuns.length);
 
   } catch (error) {
     console.error('❌ Error:', error.message);

From e64b4ffafc1496232348ca7c75a6fa189485201b Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Wed, 3 Sep 2025 15:44:45 +0200
Subject: [PATCH 06/13] ci: add flaky tests bot workflow

---
 .github/scripts/create-flaky-test-report.mjs |  4 +-
 .github/workflows/flaky-test-report.yml      | 54 ++++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/flaky-test-report.yml

diff --git a/.github/scripts/create-flaky-test-report.mjs b/.github/scripts/create-flaky-test-report.mjs
index 14a24b9c..1c8d03c9 100644
--- a/.github/scripts/create-flaky-test-report.mjs
+++ b/.github/scripts/create-flaky-test-report.mjs
@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 
+// Based on the original script done by @itsyoboieltr on Extension repo
+
 import { Octokit } from '@octokit/rest';
 import unzipper from 'unzipper';
 import { IncomingWebhook } from '@slack/webhook';
@@ -516,7 +518,7 @@ function createSlackBlocks(summary, dateDisplay, workflowCount = 0, failedCount
           elements: [{
             type: 'rich_text_section',
             elements: [
-              { type: 'text', text: `     ${errorPreview.replace(/\n/g, ' ')}` }  // 5 spaces indent for error
+              { type: 'text', text: `     ${errorPreview.replace(/\n/g, ' ')}` }
             ]
           }]
         });
diff --git a/.github/workflows/flaky-test-report.yml b/.github/workflows/flaky-test-report.yml
new file mode 100644
index 00000000..6ede74c8
--- /dev/null
+++ b/.github/workflows/flaky-test-report.yml
@@ -0,0 +1,54 @@
+name: Flaky Test Report
+
+on:
+  workflow_call:
+    inputs:
+      repository:
+        description: 'Repository name (e.g. metamask-extension)'
+        required: true
+        type: string
+      workflow_id:
+        description: 'Workflow ID to analyze (e.g. main.yml)'
+        required: true
+        type: string
+    secrets:
+      github-token:
+        description: 'GitHub token with repo and actions:read access'
+        required: true
+      slack-webhook-flaky-tests:
+        description: 'Slack webhook URL for flaky test reports'
+        required: true
+
+jobs:
+  flaky-test-report:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout github-tools repository
+        uses: actions/checkout@v4
+        with:
+          repository: MetaMask/github-tools
+          path: github-tools
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ./github-tools/.nvmrc
+          cache-dependency-path: ./github-tools/yarn.lock
+          cache: yarn
+
+      - name: Enable Corepack
+        run: corepack enable
+        working-directory: ./github-tools
+
+      - name: Install dependencies
+        working-directory: ./github-tools
+        run: yarn --immutable
+
+      - name: Run flaky test report script
+        working-directory: ./github-tools
+        env:
+          REPOSITORY: ${{ inputs.repository }}
+          WORKFLOW_ID: ${{ inputs.workflow_id }}
+          GITHUB_TOKEN: ${{ secrets.github-token }}
+          SLACK_WEBHOOK_FLAKY_TESTS: ${{ secrets.slack-webhook-flaky-tests }}
+        run: node .github/scripts/create-flaky-test-report.mjs

From 0e778994c77974b426a2653011e0c01bc2ff85c7 Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Wed, 3 Sep 2025 15:46:39 +0200
Subject: [PATCH 07/13] ci: add flaky tests bot workflow

---
 yarn.lock | 64 ++++---------------------------------------------------
 1 file changed, 4 insertions(+), 60 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index f3302485..80291c24 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1472,14 +1472,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@slack/types@npm:^2.11.0":
-  version: 2.14.0
-  resolution: "@slack/types@npm:2.14.0"
-  checksum: 10/fa24a113b88e087f899078504c2ba50ab9795f7c2dd1a2d95b28217a3af20e554494f9cc3b8c8ce173120990d98e19400c95369f9067cecfcc46c08b59d2a46f
-  languageName: node
-  linkType: hard
-
-"@slack/types@npm:^2.9.0":
+"@slack/types@npm:^2.11.0, @slack/types@npm:^2.9.0":
   version: 2.16.0
   resolution: "@slack/types@npm:2.16.0"
   checksum: 10/e18b568a47d94e9e7234dfd06f789224d6804edae4a2f31068b3f388ce4c482a6dbc6c035dc3dec63e5723f211f92c7694ee40b2ec83d4ac90d46bb35fa46eb5
@@ -1896,16 +1889,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/node@npm:*, @types/node@npm:>=12.0.0":
-  version: 22.13.1
-  resolution: "@types/node@npm:22.13.1"
-  dependencies:
-    undici-types: "npm:~6.20.0"
-  checksum: 10/d8ba7068b0445643c0fa6e4917cdb7a90e8756a9daff8c8a332689cd5b2eaa01e4cd07de42e3cd7e6a6f465eeda803d5a1363d00b5ab3f6cea7950350a159497
-  languageName: node
-  linkType: hard
-
-"@types/node@npm:>=18.0.0":
+"@types/node@npm:*, @types/node@npm:>=12.0.0, @types/node@npm:>=18.0.0":
   version: 24.3.0
   resolution: "@types/node@npm:24.3.0"
   dependencies:
@@ -2487,7 +2471,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"axios@npm:^1.11.0":
+"axios@npm:^1.11.0, axios@npm:^1.7.4":
   version: 1.11.0
   resolution: "axios@npm:1.11.0"
   dependencies:
@@ -2498,17 +2482,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"axios@npm:^1.7.4":
-  version: 1.7.9
-  resolution: "axios@npm:1.7.9"
-  dependencies:
-    follow-redirects: "npm:^1.15.6"
-    form-data: "npm:^4.0.0"
-    proxy-from-env: "npm:^1.1.0"
-  checksum: 10/b7a5f660ea53ba9c2a745bf5ad77ad8bf4f1338e13ccc3f9f09f810267d6c638c03dac88b55dae8dc98b79c57d2d6835be651d58d2af97c174f43d289a9fd007
-  languageName: node
-  linkType: hard
-
 "babel-jest@npm:^28.1.3":
   version: 28.1.3
   resolution: "babel-jest@npm:28.1.3"
@@ -3586,18 +3559,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"es-set-tostringtag@npm:^2.0.1":
-  version: 2.0.1
-  resolution: "es-set-tostringtag@npm:2.0.1"
-  dependencies:
-    get-intrinsic: "npm:^1.1.3"
-    has: "npm:^1.0.3"
-    has-tostringtag: "npm:^1.0.0"
-  checksum: 10/ec416a12948cefb4b2a5932e62093a7cf36ddc3efd58d6c58ca7ae7064475ace556434b869b0bbeb0c365f1032a8ccd577211101234b69837ad83ad204fff884
-  languageName: node
-  linkType: hard
-
-"es-set-tostringtag@npm:^2.1.0":
+"es-set-tostringtag@npm:^2.0.1, es-set-tostringtag@npm:^2.1.0":
   version: 2.1.0
   resolution: "es-set-tostringtag@npm:2.1.0"
   dependencies:
@@ -4314,17 +4276,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"form-data@npm:^4.0.0":
-  version: 4.0.1
-  resolution: "form-data@npm:4.0.1"
-  dependencies:
-    asynckit: "npm:^0.4.0"
-    combined-stream: "npm:^1.0.8"
-    mime-types: "npm:^2.1.12"
-  checksum: 10/6adb1cff557328bc6eb8a68da205f9ae44ab0e88d4d9237aaf91eed591ffc64f77411efb9016af7d87f23d0a038c45a788aa1c6634e51175c4efa36c2bc53774
-  languageName: node
-  linkType: hard
-
 "form-data@npm:^4.0.4":
   version: 4.0.4
   resolution: "form-data@npm:4.0.4"
@@ -8196,13 +8147,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"undici-types@npm:~6.20.0":
-  version: 6.20.0
-  resolution: "undici-types@npm:6.20.0"
-  checksum: 10/583ac7bbf4ff69931d3985f4762cde2690bb607844c16a5e2fbb92ed312fe4fa1b365e953032d469fa28ba8b224e88a595f0b10a449332f83fa77c695e567dbe
-  languageName: node
-  linkType: hard
-
 "undici-types@npm:~7.10.0":
   version: 7.10.0
   resolution: "undici-types@npm:7.10.0"

From 40b1fa4aae23b05868878da7dc8e497679bb60df Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Wed, 3 Sep 2025 15:55:20 +0200
Subject: [PATCH 08/13] ci: add flaky tests bot workflow

---
 .github/workflows/flaky-test-report.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/flaky-test-report.yml b/.github/workflows/flaky-test-report.yml
index 6ede74c8..e2994407 100644
--- a/.github/workflows/flaky-test-report.yml
+++ b/.github/workflows/flaky-test-report.yml
@@ -15,9 +15,6 @@ on:
       github-token:
         description: 'GitHub token with repo and actions:read access'
         required: true
-      slack-webhook-flaky-tests:
-        description: 'Slack webhook URL for flaky test reports'
-        required: true
 
 jobs:
   flaky-test-report:
@@ -50,5 +47,6 @@ jobs:
           REPOSITORY: ${{ inputs.repository }}
           WORKFLOW_ID: ${{ inputs.workflow_id }}
           GITHUB_TOKEN: ${{ secrets.github-token }}
-          SLACK_WEBHOOK_FLAKY_TESTS: ${{ secrets.slack-webhook-flaky-tests }}
+          # TODO: Temporary hardcoded secret for testing - remove this and use input secret later 
+          SLACK_WEBHOOK_FLAKY_TESTS: ${{ secrets.SLACK_WEBHOOK_FLAKY_TESTS2 }}
         run: node .github/scripts/create-flaky-test-report.mjs

From e1be32ebea30d09794fd12a8466d1cef9bc21629 Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Wed, 3 Sep 2025 15:57:26 +0200
Subject: [PATCH 09/13] ci: add flaky tests bot workflow

---
 .github/workflows/flaky-test-report.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/flaky-test-report.yml b/.github/workflows/flaky-test-report.yml
index e2994407..38d360ab 100644
--- a/.github/workflows/flaky-test-report.yml
+++ b/.github/workflows/flaky-test-report.yml
@@ -47,6 +47,6 @@ jobs:
           REPOSITORY: ${{ inputs.repository }}
           WORKFLOW_ID: ${{ inputs.workflow_id }}
           GITHUB_TOKEN: ${{ secrets.github-token }}
-          # TODO: Temporary hardcoded secret for testing - remove this and use input secret later 
+          # TODO: Temporary hardcoded secret for testing - remove this and use input secret later
           SLACK_WEBHOOK_FLAKY_TESTS: ${{ secrets.SLACK_WEBHOOK_FLAKY_TESTS2 }}
         run: node .github/scripts/create-flaky-test-report.mjs

From b21e24a247192df325ec287d0b9621781320cea5 Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Wed, 3 Sep 2025 16:10:45 +0200
Subject: [PATCH 10/13] ci: add flaky tests bot workflow

---
 .github/workflows/flaky-test-report.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/flaky-test-report.yml b/.github/workflows/flaky-test-report.yml
index 38d360ab..6ede74c8 100644
--- a/.github/workflows/flaky-test-report.yml
+++ b/.github/workflows/flaky-test-report.yml
@@ -15,6 +15,9 @@ on:
       github-token:
         description: 'GitHub token with repo and actions:read access'
         required: true
+      slack-webhook-flaky-tests:
+        description: 'Slack webhook URL for flaky test reports'
+        required: true
 
 jobs:
   flaky-test-report:
@@ -47,6 +50,5 @@ jobs:
           REPOSITORY: ${{ inputs.repository }}
           WORKFLOW_ID: ${{ inputs.workflow_id }}
           GITHUB_TOKEN: ${{ secrets.github-token }}
-          # TODO: Temporary hardcoded secret for testing - remove this and use input secret later
-          SLACK_WEBHOOK_FLAKY_TESTS: ${{ secrets.SLACK_WEBHOOK_FLAKY_TESTS2 }}
+          SLACK_WEBHOOK_FLAKY_TESTS: ${{ secrets.slack-webhook-flaky-tests }}
         run: node .github/scripts/create-flaky-test-report.mjs

From dd78732a1abaa8ffb91f2c9a9deb54e0c90a8bd3 Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Wed, 3 Sep 2025 16:39:13 +0200
Subject: [PATCH 11/13] ci: add flaky tests bot workflow

---
 .github/workflows/flaky-test-report.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/flaky-test-report.yml b/.github/workflows/flaky-test-report.yml
index 6ede74c8..1f5c298a 100644
--- a/.github/workflows/flaky-test-report.yml
+++ b/.github/workflows/flaky-test-report.yml
@@ -45,10 +45,10 @@ jobs:
         run: yarn --immutable
 
       - name: Run flaky test report script
-        working-directory: ./github-tools
         env:
           REPOSITORY: ${{ inputs.repository }}
           WORKFLOW_ID: ${{ inputs.workflow_id }}
           GITHUB_TOKEN: ${{ secrets.github-token }}
           SLACK_WEBHOOK_FLAKY_TESTS: ${{ secrets.slack-webhook-flaky-tests }}
+        working-directory: ./github-tools
         run: node .github/scripts/create-flaky-test-report.mjs

From 20312034aa9995ae1c43d907afdfa9bb32417f97 Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Wed, 3 Sep 2025 16:43:57 +0200
Subject: [PATCH 12/13] ci: add flaky tests bot workflow

---
 .github/workflows/flaky-test-report.yml | 45 +++++++++++++------------
 1 file changed, 24 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/flaky-test-report.yml b/.github/workflows/flaky-test-report.yml
index 1f5c298a..31490025 100644
--- a/.github/workflows/flaky-test-report.yml
+++ b/.github/workflows/flaky-test-report.yml
@@ -1,23 +1,26 @@
 name: Flaky Test Report
 
 on:
-  workflow_call:
-    inputs:
-      repository:
-        description: 'Repository name (e.g. metamask-extension)'
-        required: true
-        type: string
-      workflow_id:
-        description: 'Workflow ID to analyze (e.g. main.yml)'
-        required: true
-        type: string
-    secrets:
-      github-token:
-        description: 'GitHub token with repo and actions:read access'
-        required: true
-      slack-webhook-flaky-tests:
-        description: 'Slack webhook URL for flaky test reports'
-        required: true
+  push:
+    branches:
+      - ci-flaky-tests-bot
+  # workflow_call:
+  #   inputs:
+  #     repository:
+  #       description: 'Repository name (e.g. metamask-extension)'
+  #       required: true
+  #       type: string
+  #     workflow_id:
+  #       description: 'Workflow ID to analyze (e.g. main.yml)'
+  #       required: true
+  #       type: string
+  #   secrets:
+  #     github-token:
+  #       description: 'GitHub token with repo and actions:read access'
+  #       required: true
+  #     slack-webhook-flaky-tests:
+  #       description: 'Slack webhook URL for flaky test reports'
+  #       required: true
 
 jobs:
   flaky-test-report:
@@ -46,9 +49,9 @@ jobs:
 
       - name: Run flaky test report script
         env:
-          REPOSITORY: ${{ inputs.repository }}
-          WORKFLOW_ID: ${{ inputs.workflow_id }}
-          GITHUB_TOKEN: ${{ secrets.github-token }}
-          SLACK_WEBHOOK_FLAKY_TESTS: ${{ secrets.slack-webhook-flaky-tests }}
+          # REPOSITORY: ${{ inputs.repository }}
+          # WORKFLOW_ID: ${{ inputs.workflow_id }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SLACK_WEBHOOK_FLAKY_TESTS: ${{ secrets.SLACK_WEBHOOK_FLAKY_TESTS2 }}
         working-directory: ./github-tools
         run: node .github/scripts/create-flaky-test-report.mjs

From 0cda3b979c1397a728689a8b00db4af7392c6cb5 Mon Sep 17 00:00:00 2001
From: Javier Briones <1674192+jvbriones@users.noreply.github.com>
Date: Wed, 3 Sep 2025 16:46:38 +0200
Subject: [PATCH 13/13] ci: add flaky tests bot workflow

---
 .github/workflows/flaky-test-report.yml | 45 ++++++++++++-------------
 1 file changed, 21 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/flaky-test-report.yml b/.github/workflows/flaky-test-report.yml
index 31490025..1f5c298a 100644
--- a/.github/workflows/flaky-test-report.yml
+++ b/.github/workflows/flaky-test-report.yml
@@ -1,26 +1,23 @@
 name: Flaky Test Report
 
 on:
-  push:
-    branches:
-      - ci-flaky-tests-bot
-  # workflow_call:
-  #   inputs:
-  #     repository:
-  #       description: 'Repository name (e.g. metamask-extension)'
-  #       required: true
-  #       type: string
-  #     workflow_id:
-  #       description: 'Workflow ID to analyze (e.g. main.yml)'
-  #       required: true
-  #       type: string
-  #   secrets:
-  #     github-token:
-  #       description: 'GitHub token with repo and actions:read access'
-  #       required: true
-  #     slack-webhook-flaky-tests:
-  #       description: 'Slack webhook URL for flaky test reports'
-  #       required: true
+  workflow_call:
+    inputs:
+      repository:
+        description: 'Repository name (e.g. metamask-extension)'
+        required: true
+        type: string
+      workflow_id:
+        description: 'Workflow ID to analyze (e.g. main.yml)'
+        required: true
+        type: string
+    secrets:
+      github-token:
+        description: 'GitHub token with repo and actions:read access'
+        required: true
+      slack-webhook-flaky-tests:
+        description: 'Slack webhook URL for flaky test reports'
+        required: true
 
 jobs:
   flaky-test-report:
@@ -49,9 +46,9 @@ jobs:
 
       - name: Run flaky test report script
         env:
-          # REPOSITORY: ${{ inputs.repository }}
-          # WORKFLOW_ID: ${{ inputs.workflow_id }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SLACK_WEBHOOK_FLAKY_TESTS: ${{ secrets.SLACK_WEBHOOK_FLAKY_TESTS2 }}
+          REPOSITORY: ${{ inputs.repository }}
+          WORKFLOW_ID: ${{ inputs.workflow_id }}
+          GITHUB_TOKEN: ${{ secrets.github-token }}
+          SLACK_WEBHOOK_FLAKY_TESTS: ${{ secrets.slack-webhook-flaky-tests }}
         working-directory: ./github-tools
         run: node .github/scripts/create-flaky-test-report.mjs