diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 70043663..fdbc4448 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,11 +6,10 @@ on: pull_request: branches: [main, develop] workflow_call: # Allow this workflow to be called by other workflows - # Nightly browser-E2E schedule is intentionally OFF: the Playwright E2E suite is - # deferred pending a rewrite against the current Phase-3+ UI (tracked in #684). - # Re-enable this cron once those jobs are green. - # schedule: - # - cron: '0 2 * * *' + # Nightly browser-E2E schedule: runs the full Playwright suite (all browsers, + # all specs) against the current Phase-3+ UI. Rewritten in #684. + schedule: + - cron: '0 2 * * *' env: PYTHON_VERSION: '3.11' @@ -431,15 +430,145 @@ jobs: retention-days: 7 # ============================================ - # E2E Browser Tests (Playwright) — DEFERRED (see #684) + # E2E Browser Tests (Playwright) — rewritten for the Phase-3+ UI (#684) # ============================================ - # The browser-level Playwright E2E jobs (Chromium smoke + all-browsers) are - # intentionally deferred, not abandoned. The tests/e2e/*.spec.ts suite targets a - # /projects/[id] route architecture that the current Phase-3+ workspace UI no - # longer has (pages are /tasks, /execution, /proof, etc.), so it cannot pass - # as-is and needs a rewrite. Tracked in issue #684. The nightly `schedule:` cron - # at the top of this file stays off until that rewrite lands and the jobs are - # green. The prior (stale) job definitions remain available in git history. + # `playwright.config.ts` (tests/e2e) starts the backend (uv uvicorn) and the + # frontend (next build + start) itself via its `webServer` block, and + # `global-setup.ts` seeds a workspace + login user. So these jobs only install + # deps + browsers and run Playwright. + # + # - smoke: chromium, @smoke only, on every PR/push (gates merges via summary) + # - full: all browsers, all specs, nightly schedule + e2e-browser-smoke: + name: E2E Browser Smoke (Chromium) + runs-on: ubuntu-latest + needs: code-quality + permissions: + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 + with: + enable-cache: true + + - name: Install Python deps + run: | + uv venv + uv sync --extra dev + uv pip install -e . + + - name: Configure git (review diff needs a repo) + run: | + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: 'web-ui/package-lock.json' + + - name: Install frontend deps + working-directory: web-ui + run: npm ci + + - name: Install E2E deps + working-directory: tests/e2e + run: npm ci + + - name: Install Playwright browser (chromium) + working-directory: tests/e2e + run: npx playwright install --with-deps chromium + + - name: Run Playwright smoke suite + working-directory: tests/e2e + run: npx playwright test --project=chromium --grep @smoke + + - name: Upload Playwright report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: e2e-browser-smoke-report + path: tests/e2e/playwright-report/ + retention-days: 7 + + e2e-browser-full: + name: E2E Browser Full (All Browsers) + runs-on: ubuntu-latest + # Nightly only — the full cross-browser sweep is too heavy for every PR. + if: github.event_name == 'schedule' + permissions: + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 + with: + enable-cache: true + + - name: Install Python deps + run: | + uv venv + uv sync --extra dev + uv pip install -e . + + - name: Configure git (review diff needs a repo) + run: | + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: 'web-ui/package-lock.json' + + - name: Install frontend deps + working-directory: web-ui + run: npm ci + + - name: Install E2E deps + working-directory: tests/e2e + run: npm ci + + - name: Install Playwright browsers (all) + working-directory: tests/e2e + run: npx playwright install --with-deps + + - name: Run full Playwright suite + working-directory: tests/e2e + run: npx playwright test + + - name: Upload Playwright report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: e2e-browser-full-report + path: tests/e2e/playwright-report/ + retention-days: 7 # ============================================ # TestSprite E2E Tests (Optional) @@ -498,7 +627,7 @@ jobs: test-summary: name: Test Summary runs-on: ubuntu-latest - needs: [backend-tests, frontend-tests, code-quality, check-hardcoded-urls] + needs: [backend-tests, frontend-tests, code-quality, check-hardcoded-urls, e2e-browser-smoke] if: always() steps: @@ -512,10 +641,17 @@ jobs: echo "| Hardcoded URLs | ${{ needs.check-hardcoded-urls.result }} |" >> $GITHUB_STEP_SUMMARY echo "| Backend Tests | ${{ needs.backend-tests.result }} |" >> $GITHUB_STEP_SUMMARY echo "| Frontend Tests | ${{ needs.frontend-tests.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| E2E Browser Smoke | ${{ needs.e2e-browser-smoke.result }} |" >> $GITHUB_STEP_SUMMARY + # The smoke job is a merge gate: treat any non-success terminal state + # (failure / cancelled / timed_out) as a gate failure, not just + # "failure". Other jobs keep the file's existing "failure"-only check. if [ "${{ needs.code-quality.result }}" == "failure" ] || \ [ "${{ needs.check-hardcoded-urls.result }}" == "failure" ] || \ [ "${{ needs.backend-tests.result }}" == "failure" ] || \ + [ "${{ needs.e2e-browser-smoke.result }}" == "failure" ] || \ + [ "${{ needs.e2e-browser-smoke.result }}" == "cancelled" ] || \ + [ "${{ needs.e2e-browser-smoke.result }}" == "timed_out" ] || \ [ "${{ needs.frontend-tests.result }}" == "failure" ]; then echo "" >> $GITHUB_STEP_SUMMARY echo "❌ Some checks failed. Please review the logs above." >> $GITHUB_STEP_SUMMARY diff --git a/.gitignore b/.gitignore index e1de3205..dfabde03 100644 --- a/.gitignore +++ b/.gitignore @@ -93,6 +93,8 @@ tests/e2e/playwright-report/ tests/e2e/test-results/ tests/e2e/.auth/ tests/e2e/.codeframe/ +tests/e2e/.e2e-workspace/ +tests/e2e/.e2e-state.db* test_audit_report.md tests/integration/.env.integration web-ui/test-results/ diff --git a/tests/e2e/.test-db.sqlite b/tests/e2e/.test-db.sqlite deleted file mode 100644 index 025045ab..00000000 Binary files a/tests/e2e/.test-db.sqlite and /dev/null differ diff --git a/tests/e2e/APP_ISSUES_FOR_GITHUB.md b/tests/e2e/APP_ISSUES_FOR_GITHUB.md deleted file mode 100644 index 2a3e130d..00000000 --- a/tests/e2e/APP_ISSUES_FOR_GITHUB.md +++ /dev/null @@ -1,107 +0,0 @@ -# App Logic Issues Detected by E2E Tests - -**Date**: 2026-01-07 -**Context**: E2E test hardening exposed 23 failing tests. After fixing test logic errors, 1 genuine app issue remains. - -## Issue #1: WebSocket Does Not Send Messages After Connection (HIGH) - -## Issue #2: Metrics API Returns 404 for Date-Filtered Queries (MEDIUM) - -**Test**: `test_metrics_ui.spec.ts` - "should filter metrics by date range" - -**Behavior**: -- Date filter is changed from one value to another -- API request is made with date range parameters -- **API returns 404 {"detail":"Not Found"}** -- Component displays error: "Error: Request failed: 404 {"detail":"Not Found"}" - -**Expected**: -The metrics API should accept date range query parameters and return filtered data. - -**Impact**: Users cannot filter metrics by date range. - -**Files to Investigate**: -- `codeframe/ui/routers/metrics.py` - Check if date filter parameters are supported -- `web-ui/src/components/metrics/CostDashboard.tsx` - Verify query parameters being sent - -**Root Cause**: -The metrics API endpoint likely does not have query parameter handling for date filtering, or the route pattern doesn't match when parameters are included. - ---- - -## Issue #1 (Continued): WebSocket Does Not Send Messages After Connection (HIGH) - -**Test**: `test_dashboard.spec.ts` - "should receive real-time updates via WebSocket" - -**Behavior**: -- WebSocket connection is established successfully -- Frontend subscribes to project updates -- **No messages are ever received** from the backend -- Test correctly fails because a working WebSocket should send at least one message - -**Expected**: -The backend should send messages for: -1. Connection acknowledgment or subscription confirmation -2. Heartbeat/keepalive messages -3. State updates when project data changes - -**Impact**: Real-time updates do not work. Users don't see live agent status, task progress, or discovery updates without manual refresh. - -**Files to Investigate**: -- `codeframe/ui/websocket.py` - WebSocket handler -- `web-ui/src/lib/websocket.ts` - Frontend WebSocket client - -**Suggested Fix**: -1. Implement connection acknowledgment message on WebSocket connect -2. Implement periodic heartbeat messages -3. Verify WebSocket broadcast is being called when state changes - ---- - -## Test Logic Issues Fixed (For Reference) - -The following were **test logic errors**, not app bugs: - -| Issue | Fix Applied | -|-------|-------------| -| "Failed to fetch RSC payload" errors | Added to error filter (Next.js navigation transient) | -| Metrics beforeEach waiting for agent-status-panel | Changed to dashboard-header (always visible) | -| Response listeners set up after action | Set up BEFORE triggering action | -| Missing data-testid on DiscoveryProgress | Added data-testid="discovery-progress" | -| Date filter disappears during loading | Wait for cost-dashboard to reappear | - ---- - -## Creating GitHub Issues - -To create the GitHub issue for the WebSocket problem: - -```bash -gh issue create --title "WebSocket does not send messages after connection" \ - --body "## Description -The WebSocket connection is established but no messages are sent from the backend. - -## Current Behavior -- Frontend connects to WebSocket at \`/ws?token=...\` -- Connection is accepted (status code 101) -- No messages are received - -## Expected Behavior -Backend should send: -1. Connection acknowledgment -2. Heartbeat messages periodically -3. State updates when data changes - -## Affected Features -- Real-time agent status updates -- Live task progress -- Discovery question updates - -## Test Evidence -\`test_dashboard.spec.ts\` line 448 - WebSocket test requires at least one message - -## Files to Investigate -- \`codeframe/ui/websocket.py\` -- \`web-ui/src/lib/websocket.ts\`" \ - --label "bug,backend,websocket" -``` diff --git a/tests/e2e/BACKEND_CONFIG_UPDATE.md b/tests/e2e/BACKEND_CONFIG_UPDATE.md deleted file mode 100644 index 2e12c102..00000000 --- a/tests/e2e/BACKEND_CONFIG_UPDATE.md +++ /dev/null @@ -1,173 +0,0 @@ -# Playwright Configuration Update - Backend Auto-Start - -## Summary - -Updated `playwright.config.ts` to automatically start both the FastAPI backend server (port 8080) and Next.js frontend server (port 3000) before running E2E tests. - -## Changes Made - -### File Modified -- `/home/frankbria/projects/codeframe/tests/e2e/playwright.config.ts` - -### What Changed - -**Before:** -```typescript -webServer: process.env.CI - ? undefined - : { - command: 'cd ../../web-ui && npm run dev', - url: 'http://localhost:3000', - reuseExistingServer: !process.env.CI, - timeout: 120000, - }, -``` - -**After:** -```typescript -webServer: process.env.CI - ? undefined - : [ - // Backend FastAPI server - { - command: 'cd ../.. && uv run uvicorn codeframe.ui.server:app --port 8080', - url: 'http://localhost:8080/health', - reuseExistingServer: !process.env.CI, - timeout: 120000, - }, - // Frontend Next.js dev server - { - command: 'cd ../../web-ui && npm run dev', - url: 'http://localhost:3000', - reuseExistingServer: !process.env.CI, - timeout: 120000, - }, - ], -``` - -## Technical Details - -### Backend Server Configuration -- **Command**: `cd ../.. && uv run uvicorn codeframe.ui.server:app --port 8080` - - Uses `uv` package manager (per project standards) - - Starts FastAPI app from project root - - Runs on port 8080 - -- **Health Check**: `http://localhost:8080/health` - - Endpoint returns: `{"status":"healthy","service":"CodeFRAME Status Server","version":"0.1.0","commit":"634a75b","deployed_at":"...","database":"connected"}` - - Verified endpoint exists at line 262 in `codeframe/ui/server.py` - -- **Startup Sequence**: Backend starts BEFORE frontend (critical for API dependencies) - -### Frontend Server Configuration -- **Command**: `cd ../../web-ui && npm run dev` -- **URL**: `http://localhost:3000` -- **Startup**: Waits for backend to be healthy first - -## Verification - -### Automated Verification Script -Created `/home/frankbria/projects/codeframe/tests/e2e/verify-config.js` to validate: -1. TypeScript compilation of config file -2. webServer array structure (2 servers) -3. Backend server configuration (port, command, health check) -4. Frontend server configuration - -**Run verification:** -```bash -cd tests/e2e && node verify-config.js -``` - -**Output:** -``` -✅ All configuration checks passed! -``` - -### Manual Testing -1. **Backend server startup test:** - ```bash - uv run uvicorn codeframe.ui.server:app --port 8080 - curl http://localhost:8080/health - ``` - Result: Server starts successfully, health endpoint returns JSON response - -2. **TypeScript compilation:** - ```bash - cd tests/e2e && npx tsc --noEmit playwright.config.ts - ``` - Result: No errors - -3. **Playwright test listing:** - ```bash - cd tests/e2e && npx playwright test --list - ``` - Result: 120+ tests discovered across all spec files - -## Benefits - -### Before (Phase 1 - Problem) -- ❌ Backend server not auto-started -- ❌ Tests fail with connection errors to port 8080 -- ❌ Manual server startup required before running tests -- ❌ Inconsistent test environment - -### After (Phase 2 - Solution) -- ✅ Both servers auto-start before tests run -- ✅ Backend health check ensures server is ready -- ✅ Frontend waits for backend to be healthy -- ✅ Consistent, repeatable test environment -- ✅ No manual setup required - -## Environment Variables - -### CI Mode -- When `CI` env var is set, `webServer` is `undefined` -- Assumes servers are started externally in CI pipeline - -### Development Mode -- When `CI` is not set, both servers auto-start -- `reuseExistingServer: true` - Reuses running servers if already started -- `timeout: 120000` - Waits up to 2 minutes for servers to be healthy - -## Database Configuration - -The backend server uses environment variables for database configuration: -- `DATABASE_PATH` - Explicit path to state.db (optional) -- `WORKSPACE_ROOT` - Root directory for workspaces (defaults to `.codeframe/workspaces`) - -If neither is set, defaults to: -``` -.codeframe/state.db -``` - -## Next Steps - -1. **Run E2E tests:** - ```bash - cd tests/e2e - npx playwright test - ``` - -2. **Monitor server startup:** - - Backend logs will show migration status and port binding - - Frontend logs will show Next.js compilation and dev server URL - -3. **Verify global-setup.ts:** - - Ensure `BACKEND_URL` environment variable defaults to `http://localhost:8080` - - Confirm test project creation works with auto-started backend - -4. **Fix any remaining test failures:** - - Most failures should now be resolved - - Check for API endpoint mismatches (e.g., seeding endpoints) - -## Files Modified -1. `/home/frankbria/projects/codeframe/tests/e2e/playwright.config.ts` - Updated webServer configuration - -## Files Created -1. `/home/frankbria/projects/codeframe/tests/e2e/verify-config.js` - Configuration verification script -2. `/home/frankbria/projects/codeframe/tests/e2e/BACKEND_CONFIG_UPDATE.md` - This documentation - -## References -- Playwright webServer documentation: https://playwright.dev/docs/test-webserver -- FastAPI deployment: https://fastapi.tiangolo.com/deployment/manually/ -- Project CLAUDE.md: Stack preferences (uv, FastAPI, SQLite) diff --git a/tests/e2e/E2E_TEST_AUDIT.md b/tests/e2e/E2E_TEST_AUDIT.md index baf99c25..7b01975e 100644 --- a/tests/e2e/E2E_TEST_AUDIT.md +++ b/tests/e2e/E2E_TEST_AUDIT.md @@ -1,96 +1,32 @@ -# E2E Test Audit Report - -**Date**: 2026-01-09 (Updated) -**Auditor**: Claude Code -**Status**: CRITICAL ISSUES RESOLVED - -## Summary - -All critical issues identified in the original audit have been addressed. The E2E test suite now uses real JWT authentication, strict error filtering, and proper API response validation. - -## Fixes Applied - -### Authentication (RESOLVED) - -1. **Auth bypass removed** - `auth-bypass.ts` deleted -2. **Real JWT authentication** - All tests use `loginUser()` from `test-utils.ts` -3. **Lint API fixed** - Migrated from standalone axios to `authFetch` with JWT headers -4. **Response interceptor added** - `api.ts` now logs 401 errors with debugging context -5. **TaskReview error handling improved** - Extracts specific error messages, handles auth failures - -### Error Filtering (RESOLVED) - -1. **Strict filtering applied** - All test files now only filter: - - `net::ERR_ABORTED` - Normal navigation cancellation - - `Failed to fetch RSC payload` - Next.js transient during navigation -2. **WebSocket errors NOT filtered** - Connection and message failures will cause test failures -3. **API errors NOT filtered** - 401, 500, network failures will cause test failures - -### WebSocket Test (RESOLVED) - -1. **Now REQUIRES messages** - `test_dashboard.spec.ts:445-455` throws error if 0 messages -2. **Auth error detection** - Detects and reports close code 1008 (auth error) -3. **Abnormal close detection** - Detects and reports close code 1006 - -### Conditional Skips (RESOLVED) - -All conditional skips now verify alternate state before skipping: -```typescript -// Pattern used in tests: -const hasKnownState = (await alternateElement.count() > 0); -expect(hasKnownState).toBe(true); // MUST be in SOME known state -test.skip(true, 'Reason (verified in alternate state)'); -``` - -This ensures tests catch broken pages (where neither expected nor alternate state exists). - -### API Response Validation (RESOLVED) - -1. **Task approval test added** - `test_task_breakdown.spec.ts` validates 401 errors specifically -2. **Metrics tests validate responses** - `test_metrics_ui.spec.ts:28-56` -3. **Project creation validates responses** - `test_project_creation.spec.ts` - -## Current Test Architecture - -### Authentication Flow -``` -loginUser(page) -> /login page -> fill credentials -> submit -> JWT stored in localStorage - -> redirect to /projects -All subsequent API calls include Authorization: Bearer {token} header -WebSocket connections include ?token={token} query parameter -``` - -### Error Monitoring -``` -setupErrorMonitoring(page) -> captures console errors, network failures, failed requests -afterEach: checkTestErrors(page, context, [minimal filters]) -> asserts no unexpected errors -``` - -### Test File Structure - -| File | Focus | Auth Method | -|------|-------|-------------| -| `test_auth_flow.spec.ts` | Authentication flows | Real login UI | -| `test_project_creation.spec.ts` | Project CRUD | JWT via loginUser | -| `test_task_breakdown.spec.ts` | Task generation/approval | JWT via loginUser | -| `test_dashboard.spec.ts` | Dashboard + WebSocket | JWT via loginUser | -| `test_complete_user_journey.spec.ts` | End-to-end workflow | JWT via loginUser | -| `test_start_agent_flow.spec.ts` | Discovery + agents | JWT via loginUser | -| `test_metrics_ui.spec.ts` | Metrics dashboard | JWT via loginUser | -| `test_task_execution_flow.spec.ts` | Task execution | JWT via loginUser | - -## Remaining Items (Low Priority) - -1. **Console.log patterns** - Some tests log success without assertion (acceptable for debugging) -2. **toBeAttached vs toBeVisible** - Some uses are intentional (checking DOM presence before interaction) -3. **Test data fixtures** - Consider adding for more consistent project states - -## Verification - -Run full test suite to verify: -```bash -cd tests/e2e -npx playwright test --project=chromium -``` - -Expected: All tests pass with real authentication. Any 401 errors or auth failures will cause test failures. +# Browser E2E suite — status + +_Last refreshed: 2026-06-20 (issue #684)._ + +The browser E2E suite was **rewritten** against the current Phase-3+ workspace +UI. The previous suite targeted a `/projects/[id]` route architecture that no +longer exists and was deleted. + +## Current suite + +| Spec | Covers | +|------|--------| +| `smoke.spec.ts` | `@smoke` — real `/login` flow, bad-credentials, every page renders against seeded data, session persistence | +| `tasks.spec.ts` | Task board renders all seeded tasks + statuses, title search | +| `prd.spec.ts` | Seeded PRD content, Stress Test action | +| `blockers.spec.ts` | Seeded open blocker + sidebar count badge | +| `proof.spec.ts` | PROOF9 requirement list, Capture Glitch / Run Gates, requirement detail nav | +| `review.spec.ts` | Working-tree diff for the seeded git change, review actions | +| `settings.spec.ts` | All settings tabs render + switch | +| `costs.spec.ts` | Seeded spend summary, time-range selector | +| `sessions.spec.ts` | Sessions + Execution views render | + +## How it runs + +- **Smoke** (`@smoke`, chromium): every PR/push via the `e2e-browser-smoke` CI + job, gated through `test-summary`. +- **Full** (all browsers, all specs): nightly `schedule:` cron via + `e2e-browser-full`. + +`playwright.config.ts` starts the backend + frontend itself; `global-setup.ts` +seeds a workspace (`seed_workspace.py`) and writes an authenticated +storageState. See `README.md` for local runs. diff --git a/tests/e2e/README-USER-JOURNEY-TESTS.md b/tests/e2e/README-USER-JOURNEY-TESTS.md deleted file mode 100644 index deda78fa..00000000 --- a/tests/e2e/README-USER-JOURNEY-TESTS.md +++ /dev/null @@ -1,252 +0,0 @@ -# E2E User Journey Tests - Implementation Notes - -## Overview - -This document describes the implementation of comprehensive E2E tests that validate complete user journeys through actual UI interactions, rather than bypassing flows through database seeding. - -## Test Files Created - -### 1. `test_auth_flow.spec.ts` (18 test cases) -**Comprehensive authentication tests including:** -- Login page rendering -- Successful login with valid credentials -- Login failures (invalid email, invalid password, empty form) -- Logout functionality -- Session persistence across page reloads -- Session persistence across navigation -- Protected route access when authenticated -- Redirect to login when accessing protected routes unauthenticated -- FastAPI Users JWT API integration (sign-in endpoint) -- Database integration (session creation in CodeFRAME tables) - -### 2. `test_project_creation.spec.ts` (3 test cases) -- Root page display with create project option -- Creating new project via UI -- Form validation for required fields - -### 3. `test_start_agent_flow.spec.ts` (3 test cases) -- Starting Socratic discovery from dashboard -- Answering discovery questions and PRD generation -- Agent status panel verification - -### 4. `test_complete_user_journey.spec.ts` (1 comprehensive test) -- Full workflow from login → project creation → discovery → PRD → agent execution -- Dashboard panel accessibility verification -- Tab navigation validation - -## Frontend Changes - -### Data-testid Attributes Added - -The following components were updated with `data-testid` attributes for stable test selectors: - -**LoginForm.tsx:** -- `email-input` - Email input field -- `password-input` - Password input field -- `login-button` - Login submit button -- `auth-error` - Authentication error message - -**ProjectCreationForm.tsx:** -- `project-name-input` - Project name input -- `project-description-input` - Project description textarea -- `create-project-submit` - Submit button -- `form-error` - Validation error messages - -**ProjectList.tsx:** -- `create-project-button` - Create new project button -- `project-list` - Projects grid container - -**Navigation.tsx:** -- `user-menu` - User email display -- `logout-button` - Logout button - -**DiscoveryProgress.tsx:** -- `discovery-question` - Current discovery question display -- `discovery-answer-input` - Answer textarea -- `submit-answer-button` - Submit answer button - -**Dashboard.tsx:** -- `prd-generated` - View PRD button (indicates PRD exists) -- `dashboard-header` - Dashboard header -- `agent-status-panel` - Agent status panel -- `metrics-panel` - Cost & metrics panel -- `review-findings-panel` - Code review findings panel -- `checkpoint-panel` - Checkpoints panel -- `nav-menu` - Navigation tabs -- `overview-tab`, `context-tab`, `checkpoint-tab` - Tab buttons - -## Test Utilities - -### Helper Functions (`test-utils.ts`) - -**`loginUser(page, email, password)`** -- Navigates to /login -- Fills credentials -- Submits form -- Waits for redirect to root/projects page - -**`createTestProject(page, name, description)`** -- Navigates to root -- Clicks create project button -- Fills form with unique timestamped name -- Returns project ID from URL - -**`answerDiscoveryQuestion(page, answer)`** -- Waits for discovery input -- Fills answer -- Submits -- Waits for next question or completion - -## Authentication System: FastAPI Users JWT Authentication - -### Current Authentication Architecture - -The application uses FastAPI Users with JWT tokens for authentication. E2E tests use real authentication flows. - -**Implementation:** -- **Backend:** FastAPI Users module in `codeframe/auth/` -- **Frontend:** JWT token stored in `localStorage.getItem('auth_token')` -- **WebSocket:** Token included as query parameter: `?token={jwt_token}` -- **API Client:** Authenticated axios instance in `web-ui/src/lib/api.ts` - -**Test Authentication Flow:** -1. `loginUser(page)` navigates to `/login` -2. Fills email/password credentials -3. Submits form, JWT returned and stored in localStorage -4. Redirect to `/projects` confirms successful auth -5. All subsequent API calls include `Authorization: Bearer {token}` header - -**Test User Credentials:** -- Email: `test@example.com` -- Password: `Testpassword123` -- Seeded by `seed-test-data.py` into `users` table - -**E2E Test Helpers (in `test-utils.ts`):** -- `loginUser(page)` - Real login via UI -- `registerUser(page, name, email, password)` - Real signup via UI -- `isAuthenticated(page)` - Check localStorage for auth token -- `clearAuth(page)` - Remove auth token -- `getAuthToken(page)` - Get current JWT token - -**Benefits:** -- ✅ Tests validate the real authentication flow end-to-end -- ✅ 401 errors in tests indicate real authentication bugs -- ✅ No mocking or bypassing - what works in tests works in production - -## Current Status & Known Issues - -### ✅ Completed -- All frontend components have data-testid attributes -- Test utilities created -- 4 test spec files with comprehensive test cases written -- **Unified authentication system** - FastAPI Users JWT authentication -- Tests use real login flow (no more auth bypass) -- TypeScript compilation passes -- Frontend build succeeds - -### ✅ Resolved: Next.js Dev Server Timing Issue - -**Issue:** -Initially, tests failed with 404 errors when navigating to routes during E2E test execution because Next.js development server compiles pages on-demand. - -**Resolution:** -Modified `playwright.config.ts` to use **production build** for E2E tests instead of dev server. This ensures all routes are pre-compiled and available immediately. - -**Implementation:** -```typescript -webServer: [ - // Frontend - production mode (stable for E2E tests) - { - command: 'cd ../../web-ui && TEST_DB_PATH=${TEST_DB_PATH} PORT=3001 npm run build && npm start', - url: FRONTEND_URL, - reuseExistingServer: !process.env.CI, - timeout: 120000, - } -] -``` - -**Result:** All project creation tests now pass consistently across all browsers (15/15 passed). - -## Running the Tests - -### Prerequisites -1. Backend server running on port 8080 -2. Frontend server running on port 3000 (or production build) -3. Test database initialized - -### Command -```bash -cd tests/e2e -npx playwright test test_auth_flow.spec.ts test_project_creation.spec.ts test_start_agent_flow.spec.ts test_complete_user_journey.spec.ts --project=chromium -``` - -### CI/CD Considerations -- Use Option 1 (production builds) for CI environments -- Ensure sufficient timeout buffers -- Run tests sequentially (`--workers=1`) to avoid database conflicts -- Use retries (`--retries=2`) for flaky network conditions - -## Test Design Principles - -### UI-Driven vs Database Seeding -These tests intentionally interact with the actual UI rather than bypassing it through database seeding to: -- Validate the complete user experience -- Catch UI regressions and routing issues -- Test authentication flows end-to-end -- Ensure forms work as beta testers will use them - -### Session Management -Tests clear cookies before execution to: -- Start from a logged-out state -- Test actual login flows -- Avoid conflicts with global setup's pre-seeded session - -### Unique Project Names -Projects created during tests use timestamps to: -- Avoid name conflicts across test runs -- Enable parallel test execution (future) -- Simplify test data cleanup - -## Next Steps - -1. **Fix Next.js timing issue** - Implement Option 1 (production builds) for reliable test execution -2. **Verify all tests pass** - Run full suite across all browsers (Chromium, Firefox, WebKit) -3. **Add CI integration** - Update CI workflow to run user journey tests -4. **Monitor flakiness** - Track test stability over multiple runs -5. **Add test data cleanup** - Implement teardown to remove test projects - -## Acceptance Criteria Status - -| Criterion | Status | -|-----------|--------| -| 4 test files created | ✅ Complete | -| `test_auth_flow.spec.ts` with 4 tests | ✅ Complete | -| `test_project_creation.spec.ts` with 3 tests | ✅ Complete | -| `test_start_agent_flow.spec.ts` with 3 tests | ✅ Complete | -| `test_complete_user_journey.spec.ts` with 1 test | ✅ Complete | -| Helper utilities in `test-utils.ts` | ✅ Complete | -| Tests pass on Chromium, Firefox, WebKit | ✅ Complete - 15/15 project creation tests passing | -| Tests run in CI without flakiness | ✅ Complete - Real authentication flow ensures production-like testing | -| Coverage for `/login`, `/`, dashboard flows | ✅ Complete | - -## Files Modified - -### Frontend Components -- `web-ui/src/components/auth/LoginForm.tsx` -- `web-ui/src/components/ProjectCreationForm.tsx` -- `web-ui/src/components/ProjectList.tsx` -- `web-ui/src/components/Navigation.tsx` -- `web-ui/src/components/DiscoveryProgress.tsx` -- `web-ui/src/components/Dashboard.tsx` - -### Test Files (New) -- `tests/e2e/test_auth_flow.spec.ts` -- `tests/e2e/test_project_creation.spec.ts` -- `tests/e2e/test_start_agent_flow.spec.ts` -- `tests/e2e/test_complete_user_journey.spec.ts` - -### Test Utilities -- `tests/e2e/test-utils.ts` (extended) - -## Documentation -- `tests/e2e/README-USER-JOURNEY-TESTS.md` (this file) diff --git a/tests/e2e/README.md b/tests/e2e/README.md index df81d0ff..d72f8b94 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -1,840 +1,62 @@ -# CodeFRAME End-to-End Tests +# CodeFRAME E2E tests -Comprehensive E2E testing suite for validating the full CodeFRAME autonomous coding workflow. +Two independent suites live here: -## Quick Start +- **Browser E2E (Playwright)** — `*.spec.ts`, drives the Phase-3+ web UI. + Rewritten in #684 against the current workspace UI. +- **CLI E2E (pytest)** — `cli/`, exercises the CLI / engines (`-m e2e`). Run via + `uv run pytest tests/e2e/ -m e2e`. Unaffected by the browser rewrite. -Run all E2E tests with a single command (backend auto-starts): +This README covers the **browser** suite. -```bash -cd tests/e2e -npx playwright test -``` - -That's it! The backend server starts automatically on port 8080, database seeds, and all 85+ tests run across multiple browsers. - -## Overview - -This test suite validates Sprint 10 (Review & Polish) features and ensures the complete autonomous workflow functions correctly from discovery through completion. - -### Test Coverage - -**Backend E2E Tests (Pytest)**: -- ✅ Discovery phase (Socratic Q&A) -- ✅ Task generation from PRD -- ✅ Multi-agent execution and coordination -- ✅ Quality gates enforcement -- ✅ Review agent code analysis -- ✅ Checkpoint creation and restore -- ✅ Human-in-the-loop blocker resolution -- ✅ Context management (flash save) -- ✅ Session lifecycle (pause/resume) -- ✅ Cost tracking accuracy -- ✅ Complete Hello World API project - -**Frontend E2E Tests (Playwright)**: -- ✅ Dashboard displays all Sprint 10 features -- ✅ Review findings panel and severity badges -- ✅ Checkpoint UI workflow -- ✅ Metrics and cost tracking dashboard - -**Total Tests**: 21 E2E tests covering >85% of user workflows - -## Prerequisites - -### Backend Tests -- Python 3.11+ -- uv package manager -- Git (for checkpoint tests) - -### Frontend Tests -- Node.js 20+ -- npm -- Playwright browsers - -## Installation - -### Backend E2E Tests - -```bash -# From project root -uv venv -uv sync -``` - -### Frontend E2E Tests - -```bash -# From tests/e2e directory -cd tests/e2e -npm install -npm run install:browsers # Install Playwright browsers -``` - -## Running Tests - -### Backend E2E Tests - -```bash -# Run all backend E2E tests -uv run pytest tests/e2e/test_*.py -v -m "e2e" - -# Run specific test file -uv run pytest tests/e2e/test_full_workflow.py -v - -# Run specific test -uv run pytest tests/e2e/test_full_workflow.py::test_discovery_phase -v - -# Run with coverage -uv run pytest tests/e2e/ --cov=codeframe --cov-report=term -v -``` - -### Frontend E2E Tests - -**Important**: Backend server now auto-starts automatically via `webServer` config in `playwright.config.ts`. No manual server startup required! - -```bash -# From tests/e2e directory -cd tests/e2e - -# Run all Playwright tests (backend auto-starts) -npm test - -# Run in headed mode (see browser) -npm run test:headed - -# Run in debug mode (step through tests) -npm run test:debug - -# Run specific browser -npm run test:chromium -npm run test:firefox -npm run test:webkit - -# Run mobile tests -npm run test:mobile - -# View test report -npm run report -``` - -**What happens automatically**: -1. ✅ Backend server starts on port 8080 (with health check) -2. ✅ Frontend dev server starts on port 3000 -3. ✅ Database seeding runs (via global-setup.ts) -4. ✅ Tests execute across browsers -5. ✅ Servers shut down after tests complete - -**CI/CD Note**: In CI mode (`CI=true`), servers are NOT auto-started. CI must start them separately. - -## Test Structure - -### Backend Tests - -``` -tests/e2e/ -├── fixtures/ -│ └── hello_world_api/ # Test fixture project -│ ├── README.md -│ └── prd.md -├── test_full_workflow.py # Main workflow tests (T146-T155) -└── test_hello_world_project.py # Complete project test (T156) -``` - -### Frontend Tests - -``` -tests/e2e/ -├── test_dashboard.spec.ts # Dashboard UI (T157) -├── test_review_ui.spec.ts # Review findings UI (T158) -├── test_checkpoint_ui.spec.ts # Checkpoint UI (T159) -├── test_metrics_ui.spec.ts # Metrics dashboard UI (T160) -├── playwright.config.ts # Playwright configuration -└── package.json # Dependencies -``` - -## Test Markers - -Backend tests use pytest markers: - -- `@pytest.mark.e2e` - End-to-end tests -- `@pytest.mark.slow` - Tests that take >1 minute -- `@pytest.mark.asyncio` - Async tests - -Run specific marker: -```bash -uv run pytest -m "e2e and not slow" -``` - -## CI/CD Integration - -Tests run automatically in GitHub Actions (`.github/workflows/test.yml`): - -**On every push/PR**: -- Backend unit tests -- Frontend unit tests -- Code quality checks (lint, type check) - -**On main branch or nightly**: -- Backend E2E tests -- Frontend E2E tests (Playwright) -- TestSprite E2E tests (nightly only) - -### CI Configuration - -The test workflow: -1. Sets up Python and Node.js environments -2. Installs dependencies (uv, npm) -3. Starts backend server (port 8080) -4. Starts frontend server (port 3000) -5. Runs E2E tests -6. Uploads test reports and artifacts -7. Stops servers - -## Test Fixtures - -### Hello World API - -A minimal REST API project used for full workflow testing: - -**Endpoints**: -- `GET /health` - Health check -- `GET /hello` - Simple greeting -- `GET /hello/{name}` - Personalized greeting - -**Purpose**: -- Validate full autonomous workflow -- Test quality gates enforcement -- Test checkpoint functionality -- Complete in <5 minutes - -See `fixtures/hello_world_api/README.md` for details. - -## Environment Variables - -Backend tests: -```bash -export BACKEND_URL="http://localhost:8080" # Default -export FRONTEND_URL="http://localhost:3000" # Default -``` - -Frontend tests (via Playwright): -```bash -export FRONTEND_URL="http://localhost:3000" # Default -export CI=true # On CI (disables local dev server) -``` - -## Debugging - -### Backend Tests - -```bash -# Run with verbose output -uv run pytest tests/e2e/ -vv - -# Run with print statements visible -uv run pytest tests/e2e/ -s - -# Stop on first failure -uv run pytest tests/e2e/ -x - -# Run last failed tests -uv run pytest tests/e2e/ --lf -``` - -### Frontend Tests - -```bash -# Debug mode (step through tests) -cd tests/e2e -npm run test:debug - -# Headed mode (see browser) -npm run test:headed - -# View trace for failed tests -npx playwright show-trace playwright-report/trace.zip -``` - -## Test Reports - -### Backend Test Reports - -After running tests: -```bash -# View coverage report -coverage report -coverage html -open htmlcov/index.html # Mac -xdg-open htmlcov/index.html # Linux -``` - -### Frontend Test Reports - -After running Playwright tests: -```bash -cd tests/e2e -npm run report -# Opens HTML report in browser -``` - -Reports include: -- Test results (pass/fail) -- Screenshots on failure -- Videos on failure -- Trace files for debugging - -## Workflow Coverage Analysis - -The E2E tests cover >85% of user workflows as defined in the specification: - -| Workflow | Coverage | Tests | -|----------|----------|-------| -| Discovery → PRD | 100% | T146, T147 | -| Multi-agent execution | 100% | T148, T156 | -| Quality gates | 100% | T149 | -| Review agent | 100% | T150 | -| Checkpoint/restore | 100% | T151 | -| Blocker resolution | 100% | T152 | -| Context management | 100% | T153 | -| Session lifecycle | 100% | T154 | -| Cost tracking | 100% | T155 | -| Dashboard UI | 90% | T157 | -| Review UI | 90% | T158 | -| Checkpoint UI | 90% | T159 | -| Metrics UI | 90% | T160 | - -**Overall Coverage**: 95% of user workflows - -## Test Execution Time - -**Backend E2E Tests**: -- Individual tests: 5-30 seconds each -- `test_complete_hello_world`: 10-15 minutes (full project) -- Total suite: ~20-25 minutes - -**Frontend E2E Tests**: -- Individual tests: 10-30 seconds each -- Total suite (all browsers): ~5-10 minutes -- Single browser: ~2-3 minutes - -## Known Issues and Limitations - -### Backend Tests - -1. **Git required**: Checkpoint tests require git to be installed and configured -2. **Async tests**: Some tests may be flaky due to timing issues (use retries in CI) -3. **Long-running tests**: `test_complete_hello_world` takes 10-15 minutes - -### Frontend Tests - -1. **Server dependency**: Tests require both backend and frontend servers running -2. **Browser compatibility**: Some tests may behave differently across browsers -3. **Timing issues**: Real-time WebSocket tests may be flaky (use `waitForTimeout` carefully) - -## Troubleshooting - -### Port 8080 already in use - -**Symptom**: Backend server fails to start with "Address already in use" error. - -**Solution**: -```bash -# Find process using port 8080 -lsof -ti:8080 | xargs kill -9 - -# Or manually check and kill -lsof -i:8080 -kill -``` - -### Backend health check timeout - -**Symptom**: Playwright times out waiting for backend server to be ready. - -**Solution**: -```bash -# Check if backend can start manually -# AUTH_SECRET is required: auth is ON by default and the server refuses to -# start on the default JWT secret (issue #643). -cd /home/frankbria/projects/codeframe -AUTH_SECRET=local-e2e-test-secret uv run uvicorn codeframe.ui.server:app --port 8080 - -# If successful, check health endpoint -curl http://localhost:8080/health - -# Should return: {"status": "ok"} -``` - -### WebSocket Connection Issues - -**Symptom**: E2E test "should receive real-time updates via WebSocket" fails with `ERR_CONNECTION_REFUSED` or timeout. - -**WebSocket Health Check**: - -Playwright now waits for the WebSocket health endpoint (`/ws/health`) before starting tests. This ensures the WebSocket server is fully ready. +## Run it locally ```bash -# Verify WebSocket health endpoint -curl http://localhost:8080/ws/health - -# Should return: {"status": "ready"} -``` - -**Troubleshooting Steps**: - -1. **Check WebSocket endpoint accessibility**: - ```bash - # If /ws/health returns 404, the WebSocket router may not be mounted - # Check codeframe/ui/server.py includes the websocket router - ``` - -2. **Test WebSocket connection manually**: - ```bash - # Use the test script - uv run python scripts/test-websocket.py - - # Expected output: - # ✅ Backend is healthy - # ✅ WebSocket endpoint is ready - # ✅ WebSocket connection established - # ✅ WebSocket message exchange successful - ``` - -3. **Check browser console during tests**: - ```bash - # Run tests in headed mode to see browser - cd tests/e2e - npx playwright test test_dashboard.spec.ts -g "WebSocket" --headed - - # Check browser DevTools Network tab (WS filter) for connection errors - ``` - -4. **Verify timing**: - - Backend startup: Playwright waits up to 120s for `/ws/health` - - WebSocket connection: Test waits up to 15s for connection event - - If still failing, increase timeouts in `test_dashboard.spec.ts` - -**Common Causes**: - -- **Backend not fully initialized**: The WebSocket server needs time to start after HTTP endpoints -- **CORS issues**: Ensure WebSocket connections are allowed from frontend origin -- **Proxy interference**: If using a proxy, ensure WebSocket upgrade headers are forwarded -- **Firewall blocking**: Check that port 8080 WebSocket connections are allowed - -**Helper Functions**: - -The E2E test includes two helper functions for robust WebSocket testing: - -- `waitForWebSocketReady(baseURL)`: Polls `/ws/health` until ready (30s timeout) -- `waitForWebSocketConnection(page)`: Waits for Dashboard UI to load (10s timeout) - -These ensure the test only proceeds when WebSocket infrastructure is fully operational. - -### Database seeding errors - -**Symptom**: Tests fail with "table already exists" or foreign key errors. - -**Solution**: -```bash -# Remove test databases -rm -f tests/e2e/fixtures/*/test_state.db -rm -f .codeframe/test_state.db - -# Re-run tests (seeding happens automatically) cd tests/e2e -npx playwright test -``` - -**Note**: UNIQUE constraint warnings like `UNIQUE constraint failed: projects.id` are **expected** during seeding and harmless. These occur when seed data already exists. - -### Frontend server timeout - -**Symptom**: Tests timeout waiting for frontend dev server on port 3000. - -**Solution**: -```bash -# Ensure web-ui dependencies are installed -cd web-ui -npm install - -# Try starting frontend manually -npm run dev -``` - -### Playwright browsers not installed - -**Symptom**: Error message "Executable doesn't exist at /chromium". - -**Solution**: -```bash -cd tests/e2e -npm run install:browsers -``` - -### "Database locked" errors - -**Symptom**: SQLite database locked errors during tests. - -**Solution**: -```bash -# Stop all processes using the database -pkill -f "codeframe" -pkill -f "uvicorn" - -# Remove test databases and restart -rm -f tests/e2e/fixtures/*/test_state.db -npx playwright test -``` - -## Contributing - -When adding new E2E tests: - -1. **Follow naming convention**: `test_*.py` for backend, `*.spec.ts` for frontend -2. **Use markers**: Add `@pytest.mark.e2e` for backend tests -3. **Add documentation**: Update this README with new tests -4. **Update tasks.md**: Mark tasks as completed -5. **Test locally**: Run tests locally before pushing -6. **CI validation**: Ensure tests pass in CI - -## Error Monitoring - -All E2E tests include comprehensive error monitoring to catch issues that DOM-only testing would miss. - -### Setting Up Error Monitoring - -```typescript -import { - setupErrorMonitoring, - assertNoNetworkErrors, - ErrorMonitor -} from './test-utils'; - -test.beforeEach(async ({ page }) => { - const errorMonitor = setupErrorMonitoring(page); - (page as any).__errorMonitor = errorMonitor; -}); - -test.afterEach(async ({ page }) => { - const errorMonitor = (page as any).__errorMonitor as ErrorMonitor; - if (errorMonitor) { - assertNoNetworkErrors(errorMonitor, 'Test context'); - } -}); -``` - -### What Gets Monitored - -| Monitor | Description | Why It Matters | -|---------|-------------|----------------| -| Console errors | JavaScript errors, network failures | Catches issues invisible in DOM | -| Network errors | net::ERR_*, CORS, connection refused | Identifies backend connectivity | -| Failed requests | HTTP request failures | Catches API endpoint issues | -| WebSocket close codes | Auth errors (1008), abnormal close (1006) | Validates real-time connection | - -### API Response Validation - -Use `waitForAPIResponse` instead of `withOptionalWarning` for strict API verification: - -```typescript -// BAD: Silently ignores failures (test always passes) -await withOptionalWarning(page.waitForResponse(...), 'API'); - -// GOOD: Fails if API doesn't respond correctly -const response = await waitForAPIResponse( - page, - '/api/projects/1', - { expectedStatus: 200 } -); -expect(response.data.id).toBeDefined(); -``` - -### WebSocket Monitoring - -```typescript -const wsMonitor = await monitorWebSocket(page, { - timeout: 15000, - minMessages: 1 // Expect at least 1 message -}); -assertWebSocketHealthy(wsMonitor); -``` +npm ci +npx playwright install --with-deps chromium # add firefox webkit for the full run -**WebSocket Close Codes**: -- `1000`: Normal closure (OK) -- `1006`: Abnormal closure (connection lost) -- `1008`: Policy violation (auth error - check token) +# Smoke (chromium, fast) — what PRs run: +npm run test:smoke -## Best Practices - -### General - -1. **Keep tests focused**: Each test should validate one workflow -2. **Use fixtures**: Reuse setup code with pytest/Playwright fixtures -3. **Clean up resources**: Ensure temporary files/databases are cleaned up -4. **Handle async properly**: Use `await` for all async operations -5. **Avoid hardcoded waits**: Use `waitFor*` methods instead of `sleep()` -6. **Test data isolation**: Each test should use independent test data -7. **Descriptive assertions**: Use clear assertion messages -8. **Document test purpose**: Add docstrings explaining what each test validates -9. **Backend auto-start**: Rely on `webServer` config in Playwright (don't manually start backend) -10. **Health endpoints**: Ensure backend `/health` endpoint responds quickly for Playwright health checks - -### Strict Testing Patterns - -11. **Always verify API responses return data**, not just status codes: - ```typescript - const response = await waitForAPIResponse(page, '/api/data'); - expect(response.data.items).toBeInstanceOf(Array); - ``` - -12. **Use strict assertions** - avoid `>=0` or optional checks: - ```typescript - // BAD: Always passes - expect(messages.length).toBeGreaterThanOrEqual(0); - - // GOOD: Actual validation - expect(messages.length).toBeGreaterThan(0); - ``` - -13. **Monitor console errors** - network failures should fail tests: - ```typescript - test.afterEach(async ({ page }) => { - const monitor = (page as any).__errorMonitor; - assertNoNetworkErrors(monitor); - }); - ``` - -14. **Use environment variables** for URLs (never hardcode localhost): - ```typescript - // BAD - const API_URL = 'http://localhost:8080'; - - // GOOD - const API_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8080'; - ``` - -## State Reconciliation Testing - -State reconciliation tests validate that the UI correctly reflects backend state for "late-joining users" who navigate to a project AFTER events have occurred (missing WebSocket events). - -### The Problem - -Components like `DiscoveryProgress.tsx` rely on WebSocket events to update state: -- Users present during events see correct state via WebSocket -- Users who join late (page refresh, new tab, login after events) miss these events -- Without proper state reconciliation, late-joining users see incorrect UI (e.g., "Generate Tasks" button when tasks already exist) - -### The Solution - -1. **Components check API state on mount** (not just rely on WebSocket) -2. **State initialization flags** prevent UI flash during async checks -3. **Tests navigate to pre-seeded projects** and verify UI without WebSocket events - -### Test Projects - -Five test projects are seeded with different lifecycle states (see `seed-test-data.py`): - -| Project ID | Phase | State Description | -|------------|-------|-------------------| -| 1 | discovery | Active discovery questions | -| 2 | planning | PRD complete, tasks generated | -| 3 | active | Agents working, tasks in progress | -| 4 | review | Tasks complete, quality gates run | -| 5 | completed | All work done | - -Use `TEST_PROJECT_IDS` from `e2e-config.ts`: -```typescript -import { TEST_PROJECT_IDS } from './e2e-config'; - -const projectId = TEST_PROJECT_IDS.PLANNING; -await page.goto(`${FRONTEND_URL}/projects/${projectId}`); -``` - -### Writing State Reconciliation Tests - -**Pattern**: Navigate to pre-seeded project, verify UI matches backend state without WebSocket events. - -```typescript -test('should show correct state when X already completed', async ({ page }) => { - // Use pre-seeded project in specific state - const projectId = TEST_PROJECT_IDS.PLANNING; - - // Navigate as "late-joining user" (fresh page load, no WebSocket history) - await page.goto(`${FRONTEND_URL}/projects/${projectId}`); - await page.waitForLoadState('networkidle'); - - // Wait for dashboard to load - await page.locator('[data-testid="dashboard-header"]').waitFor({ - state: 'visible', - timeout: 15000, - }); - - // CRITICAL: Verify UI matches backend state - // Correct element should be visible - await expect(page.locator('[data-testid="x-completed"]')).toBeVisible(); - - // Incorrect element should NOT be visible - await expect(page.locator('[data-testid="x-button"]')).not.toBeVisible(); -}); +# Full suite (all installed browsers, all specs) — what nightly runs: +npm run test:full ``` -### Anti-Patterns to Avoid - -1. **Conditional skips that accept ANY alternate state**: - ```typescript - // BAD: Masks bugs by accepting any state - if (!buttonVisible) { - test.skip(true, 'Button not visible'); - return; - } - - // GOOD: Verify project is in expected state before testing - const { phase } = await getProjectPhase(request, token, projectId); - expect(phase).toBe('planning'); - ``` - -2. **Tests that only work when user is present during entire workflow**: - ```typescript - // BAD: Relies on WebSocket events - await page.waitForEvent('websocket-message'); - - // GOOD: Check API state directly - const tasksResponse = await request.get(`${API}/projects/${id}/tasks`); - expect(tasksResponse.data.total).toBeGreaterThan(0); - ``` - -3. **Relying on WebSocket events without API state checks**: - ```typescript - // BAD: Component only updates via WebSocket - wsClient.onMessage((msg) => setTasksGenerated(true)); +`playwright.config.ts` starts everything for you: the FastAPI backend +(`uv run uvicorn`, port 8080) and the Next.js frontend (`next build && start`, +port 3001). Nothing else needs to be running. Override ports/URLs with +`E2E_BACKEND_PORT`, `E2E_BACKEND_URL`, `E2E_FRONTEND_URL` if 8080/3001 are taken. - // GOOD: Component checks API on mount AND listens to WebSocket - useEffect(() => { - fetchTasks().then(tasks => setTasksGenerated(tasks.length > 0)); - }, []); - wsClient.onMessage((msg) => setTasksGenerated(true)); - ``` +## How it works -### State Reconciliation Test Files - -- `test_state_reconciliation.spec.ts` - Comprehensive state reconciliation tests -- `test_late_joining_user.spec.ts` - Late-joining user scenarios (may catch WebSocket events) -- `test_returning_user.spec.ts` - Returning user scenarios (no WebSocket events) - -## Returning User vs Late-Joining User - -**Critical distinction** (GitHub Issue #231): - -| Scenario | WebSocket | Data Source | Test Pattern | -|----------|-----------|-------------|--------------| -| **Late-Joining** | May catch some events | API + partial WebSocket | Navigate during active session | -| **Returning User** | No events received | API only | Block WebSocket, navigate to seeded project | - -### The Returning User Problem (Fixed in #231) - -Users who navigate to a project AFTER all events occurred (page refresh, login later, new tab) don't receive WebSocket history. Before the fix: - -```typescript -// OLD BEHAVIOR: Tasks only loaded via WebSocket events -useEffect(() => { - // Intentionally empty - tasks managed via WebSocket -}, [tasksData]); -``` - -After the fix: - -```typescript -// NEW BEHAVIOR: Tasks loaded from API on mount -useEffect(() => { - if (tasksData?.data?.tasks) { - dispatch({ type: 'TASKS_LOADED', payload: tasksData.data.tasks }); - } -}, [tasksData]); -``` - -### Writing Returning User Tests - -Block WebSocket to ensure tests don't rely on real-time events: - -```typescript -import { blockWebSocketConnections } from './test-utils'; - -test('should show state when returning to project', async ({ page }) => { - // Block WebSocket BEFORE navigation - const unblock = await blockWebSocketConnections(page); - - // Navigate as returning user (no WebSocket history) - await page.goto(`${FRONTEND_URL}/projects/${PROJECT_ID}`); - - // Wait for API data to load - await page.waitForLoadState('networkidle'); - - // Verify UI shows correct state from API - await expect(page.locator('[data-testid="task-card"]')).toHaveCount(5); - - // Cleanup - await unblock(); -}); -``` - -### Helper Functions - -Use these utilities from `test-utils.ts`: - -```typescript -// Block WebSocket connections -const unblock = await blockWebSocketConnections(page); - -// Verify task state from API -await verifyTaskStateFromAPI(page, projectId, { - inProgress: 2, - completed: 3, - total: 5, -}); - -// Verify task state from DOM -const { actualCounts, passed, errors } = await verifyTaskStateFromDOM(page, { - inProgress: 2, - completed: 3, -}); - -// Verify project phase -await verifyProjectPhaseFromAPI(page, projectId, 'active'); - -// Verify project completion -const { isComplete, hasActiveWork } = await verifyProjectCompletionFromDOM(page); -``` - -### Smoke Tests - -State reconciliation smoke tests are tagged with `@smoke`: -```bash -npm run test:smoke # Runs all @smoke tests -``` +`global-setup.ts` runs once before the specs: -Key smoke tests: -- `should show "Review Tasks" when tasks already exist @smoke` -- `should show "View PRD" when PRD already complete @smoke` -- `should maintain correct state after page refresh @smoke` +1. Wipes + recreates a throwaway workspace at `tests/e2e/.e2e-workspace`. +2. Seeds deterministic data via `seed_workspace.py` — a PRD, six tasks across + every status, a blocker, a PROOF9 requirement, token-usage rows for the Costs + page, a git working-tree diff for the Review page, and the JWT login user. +3. Logs in through the real `/auth/jwt/login` endpoint. +4. Writes an authenticated `storageState` (`auth_token` + selected workspace + path) that the specs reuse — so they start signed in with data on screen. -## References +The `smoke.spec.ts` `@smoke` auth tests start from a clean (unauthenticated) +browser and exercise the real `/login` flow. -- [Pytest Documentation](https://docs.pytest.org/) -- [Playwright Documentation](https://playwright.dev/) -- [CodeFRAME Specification](../../specs/015-review-polish/spec.md) -- [TestSprite Integration Guide](../../testsprite_tests/TESTSPRITE_INTEGRATION_GUIDE.md) +## Files -## Support +| File | Role | +|------|------| +| `playwright.config.ts` | Projects (chromium/firefox/webkit), webServer, storageState | +| `global-setup.ts` | Seed + login + write storageState | +| `seed_workspace.py` | Deterministic backend seeding (headless core APIs) | +| `e2e-env.ts` | Shared paths/URLs/keys (env-overridable) | +| `helpers.ts` | Page list, `gotoPage`, console-error guard | +| `*.spec.ts` | Smoke + per-feature specs (see `E2E_TEST_AUDIT.md`) | -For issues or questions about E2E tests: -1. Check existing tests for examples -2. Review this README -3. Consult the specification (`specs/015-review-polish/spec.md`) -4. Ask in project discussions +## CI ---- +- `e2e-browser-smoke` — chromium `@smoke`, every PR/push, gated via `test-summary`. +- `e2e-browser-full` — all browsers, all specs, nightly `schedule:` cron. -**Last Updated**: 2025-11-23 -**Test Suite Version**: 1.0 -**Status**: ✅ Complete - All E2E tests implemented +Both live in `.github/workflows/test.yml`. diff --git a/tests/e2e/blockers.spec.ts b/tests/e2e/blockers.spec.ts new file mode 100644 index 00000000..cda4ba27 --- /dev/null +++ b/tests/e2e/blockers.spec.ts @@ -0,0 +1,23 @@ +/** + * Blockers page feature coverage (issue #684, nightly suite). + */ +import { test, expect } from '@playwright/test'; +import { gotoPage, trackConsoleErrors } from './helpers'; + +const SEEDED_QUESTION = 'Which database should we use for the dashboard?'; + +test.describe('Blockers page', () => { + test('lists the seeded open blocker', async ({ page }) => { + const errors = trackConsoleErrors(page); + await gotoPage(page, '/blockers'); + await expect(page.getByText(SEEDED_QUESTION).first()).toBeVisible(); + errors.assertClean(); + }); + + test('blocker count badge appears in the sidebar', async ({ page }) => { + await gotoPage(page, '/blockers'); + // The sidebar shows an open-blocker count badge (seeded: 1). + const blockersNav = page.getByRole('link', { name: /blockers/i }); + await expect(blockersNav).toContainText(/1/); + }); +}); diff --git a/tests/e2e/browser-config.ts b/tests/e2e/browser-config.ts deleted file mode 100644 index be359bfb..00000000 --- a/tests/e2e/browser-config.ts +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Browser-Specific Configuration for E2E Tests - * - * This module centralizes all browser-specific settings, timeouts, and quirk flags. - * Import these configurations in test utilities and test files to handle cross-browser - * differences consistently. - * - * Key browser differences addressed: - * - Firefox: Slower CSS rendering, NS_BINDING_ABORTED errors during navigation - * - WebKit: Delayed element rendering, localStorage timing issues - * - Mobile: Touch events required, smaller viewports need scroll handling - */ - -/** - * Browser-specific timeout configurations - * - * Chromium is the baseline. Other browsers have multipliers applied based on - * observed performance characteristics. - * - * - Firefox: +50% for CSS rendering and form validation - * - WebKit: +40% for element stabilization and animations - * - Mobile: +50% for touch event registration and viewport adjustments - */ -export const BROWSER_TIMEOUTS = { - chromium: { - action: 10000, // Default actionTimeout - expect: 5000, // Default expect timeout - navigation: 30000, // Page navigation timeout - formValidation: 3000, - animation: 500, - }, - firefox: { - action: 15000, // +50% for slower CSS rendering - expect: 8000, // +60% for async form validation - navigation: 45000, // +50% for network handling - formValidation: 5000, // Firefox renders validation messages asynchronously - animation: 800, // CSS transitions take longer - }, - webkit: { - action: 14000, // +40% for element stabilization - expect: 7000, // +40% for delayed rendering - navigation: 40000, // +33% for Safari's network stack - formValidation: 4000, - animation: 700, // WebKit animation timing differences - }, - mobile: { - action: 15000, // +50% for touch event registration - expect: 10000, // +100% for viewport stabilization - navigation: 60000, // +100% for mobile network handling - formValidation: 5000, - animation: 1000, // Mobile animations may be slower - }, -} as const; - -/** - * Browser quirk flags - * - * These flags indicate which workarounds are needed for each browser. - * Use these to conditionally apply browser-specific handling in tests. - */ -export const BROWSER_QUIRKS = { - firefox: { - /** Firefox needs extra wait for async form validation rendering */ - needsFormValidationWait: true, - /** Firefox's NS_BINDING_ABORTED error during navigation is benign */ - hasNSBindingAborted: true, - /** Firefox may need reducedMotion for consistent animation timing */ - needsReducedMotion: true, - /** Firefox click events may need explicit wait for element stability */ - needsClickStability: false, - /** Firefox localStorage is synchronous but needs reload for visibility */ - hasDelayedLocalStorage: false, - }, - webkit: { - /** WebKit elements may not be stable immediately after appearing */ - needsElementStabilityWait: true, - /** WebKit localStorage writes may not be immediately readable */ - hasDelayedLocalStorage: true, - /** WebKit forms need click-then-fill pattern for reliable input */ - needsClickBeforeFill: true, - /** WebKit animations need explicit completion wait */ - needsAnimationWait: true, - /** WebKit needs extra time after navigation for DOM stability */ - needsPostNavigationWait: true, - }, - mobile: { - /** Mobile browsers require touch events instead of mouse clicks */ - needsTouchEvents: true, - /** Mobile viewports need scroll into view before interaction */ - requiresScrollIntoView: true, - /** Mobile may have hamburger menu instead of full navigation */ - hasResponsiveMenu: true, - /** Mobile viewport may need stabilization after orientation/resize */ - needsViewportStabilization: true, - /** Some features are desktop-only (hover states, etc.) */ - hasLimitedFeatures: true, - }, - chromium: { - /** Chromium is the baseline - no special handling needed */ - needsFormValidationWait: false, - hasNSBindingAborted: false, - needsElementStabilityWait: false, - hasDelayedLocalStorage: false, - needsTouchEvents: false, - requiresScrollIntoView: false, - }, -} as const; - -/** - * Error patterns to filter by browser - * - * These are errors that appear in specific browsers but are not actual failures. - * Use with filterExpectedErrors() in test-utils.ts. - */ -export const BROWSER_EXPECTED_ERRORS = { - firefox: [ - 'NS_BINDING_ABORTED', // Normal during navigation - 'AbortError', // Request abort during navigation - 'NetworkError when attempting', // Sometimes appears during fast navigation - ], - webkit: [ - 'Failed to load resource', // Sometimes appears during rapid navigation - 'Load request cancelled', // WebKit's equivalent of NS_BINDING_ABORTED - 'cancelled', // Generic cancellation error - ], - mobile: [ - 'touch-action', // Touch action warnings - ], - chromium: [], // Baseline - no special filtering - all: [ - 'net::ERR_ABORTED', // Normal when navigation cancels pending requests - 'Failed to fetch RSC payload', // Next.js RSC during navigation - ], -} as const; - -/** - * Mobile device viewport configurations - * - * These match Playwright's device definitions but are exported here for - * custom viewport handling in tests. - */ -export const MOBILE_VIEWPORTS = { - 'Mobile Chrome': { width: 393, height: 851, isMobile: true, hasTouch: true }, - 'Mobile Safari': { width: 390, height: 844, isMobile: true, hasTouch: true }, - 'Pixel 5': { width: 393, height: 851, isMobile: true, hasTouch: true }, - 'iPhone 12': { width: 390, height: 844, isMobile: true, hasTouch: true }, - 'iPhone 13': { width: 390, height: 844, isMobile: true, hasTouch: true }, - 'Galaxy S21': { width: 360, height: 800, isMobile: true, hasTouch: true }, -} as const; - -/** - * Browser project names as used in Playwright config - */ -export const BROWSER_PROJECTS = { - CHROMIUM: 'chromium', - FIREFOX: 'firefox', - WEBKIT: 'webkit', - MOBILE_CHROME: 'Mobile Chrome', - MOBILE_SAFARI: 'Mobile Safari', -} as const; - -export type BrowserName = 'chromium' | 'firefox' | 'webkit'; -export type MobileProjectName = 'Mobile Chrome' | 'Mobile Safari'; -export type ProjectName = BrowserName | MobileProjectName; diff --git a/tests/e2e/costs.spec.ts b/tests/e2e/costs.spec.ts new file mode 100644 index 00000000..df29eb07 --- /dev/null +++ b/tests/e2e/costs.spec.ts @@ -0,0 +1,28 @@ +/** + * Costs page feature coverage (issue #684, nightly suite). + * Seeded: 3 token_usage rows (~$0.054 total) across claude-code / codex. + */ +import { test, expect } from '@playwright/test'; +import { gotoPage, trackConsoleErrors } from './helpers'; + +test.describe('Costs page', () => { + test('shows seeded spend summary', async ({ page }) => { + const errors = trackConsoleErrors(page); + await gotoPage(page, '/costs'); + // Stable data-testids from the costs cards (#557). + await expect(page.getByTestId('total-spend')).toBeVisible(); + await expect(page.getByTestId('total-tasks')).toBeVisible(); + // Seeded total is non-zero. + await expect(page.getByTestId('total-spend')).toContainText(/\$0\.0[0-9]/); + errors.assertClean(); + }); + + test('time-range selector switches the range', async ({ page }) => { + await gotoPage(page, '/costs'); + // Native