diff --git a/codeframe/lib/metrics_tracker.py b/codeframe/lib/metrics_tracker.py index b7e2816e..fe5294ab 100644 --- a/codeframe/lib/metrics_tracker.py +++ b/codeframe/lib/metrics_tracker.py @@ -35,7 +35,7 @@ """ import logging -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Dict, Any, Optional from codeframe.core.models import CallType, TokenUsage from codeframe.persistence.database import Database @@ -202,14 +202,21 @@ async def record_token_usage( return usage_id - async def get_project_costs(self, project_id: int) -> Dict[str, Any]: + async def get_project_costs( + self, + project_id: int, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, + ) -> Dict[str, Any]: """Get total costs and breakdown for a project. Aggregates all token usage records for the project and provides - breakdowns by agent and model. + breakdowns by agent and model. Optionally filter by date range. Args: project_id: Project ID to get costs for + start_date: Optional start of date range (inclusive) + end_date: Optional end of date range (inclusive) Returns: Dictionary with cost breakdown: @@ -234,8 +241,10 @@ async def get_project_costs(self, project_id: int) -> Dict[str, Any]: >>> for agent in costs['by_agent']: ... print(f" {agent['agent_id']}: ${agent['cost_usd']:.2f}") """ - # Get all usage records for project - usage_records = self.db.get_token_usage(project_id=project_id) + # Get usage records for project (optionally filtered by date) + usage_records = self.db.get_token_usage( + project_id=project_id, start_date=start_date, end_date=end_date + ) # Initialize result result = { @@ -458,3 +467,136 @@ async def get_token_usage_stats( # This would group usage by date for timeline visualization return result + + async def get_token_usage_timeseries( + self, + project_id: int, + start_date: datetime, + end_date: datetime, + interval: str = "day", + ) -> list[dict[str, Any]]: + """Get token usage aggregated by time intervals for charting. + + Groups token usage records into time buckets (hour, day, or week) for + visualization in time series charts. Each bucket contains aggregated + token counts and costs. + + Args: + project_id: Project ID to get time series for + start_date: Start of date range (inclusive) + end_date: End of date range (inclusive) + interval: Time interval for grouping ('hour', 'day', 'week') + + Returns: + List of time series data points, each containing: + { + "timestamp": str (ISO 8601 format), + "input_tokens": int, + "output_tokens": int, + "total_tokens": int, + "cost_usd": float + } + + Raises: + ValueError: If interval is not one of 'hour', 'day', 'week' + + Example: + >>> from datetime import datetime, timedelta + >>> start = datetime.now() - timedelta(days=7) + >>> end = datetime.now() + >>> series = await tracker.get_token_usage_timeseries( + ... project_id=1, + ... start_date=start, + ... end_date=end, + ... interval='day' + ... ) + >>> for point in series: + ... print(f"{point['timestamp']}: {point['total_tokens']} tokens") + """ + valid_intervals = ("hour", "day", "week") + if interval not in valid_intervals: + raise ValueError( + f"Invalid interval '{interval}'. Must be one of: {', '.join(valid_intervals)}" + ) + + # Get usage records with date filtering + usage_records = self.db.get_token_usage( + project_id=project_id, start_date=start_date, end_date=end_date + ) + + if not usage_records: + return [] + + # Group records by time bucket + buckets: dict[str, dict[str, Any]] = {} + + for record in usage_records: + # Parse timestamp - handle string, naive datetime, and aware datetime + timestamp = record["timestamp"] + if isinstance(timestamp, str): + # Handle both ISO 8601 and simple date formats + timestamp = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + elif timestamp.tzinfo is None: + # Assume UTC for naive datetimes from database + timestamp = timestamp.replace(tzinfo=timezone.utc) + + # Calculate bucket key based on interval + bucket_key = self._get_bucket_key(timestamp, interval) + + # Initialize bucket if not exists + if bucket_key not in buckets: + buckets[bucket_key] = { + "timestamp": bucket_key, + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "cost_usd": 0.0, + } + + # Aggregate values + buckets[bucket_key]["input_tokens"] += record["input_tokens"] + buckets[bucket_key]["output_tokens"] += record["output_tokens"] + buckets[bucket_key]["total_tokens"] += ( + record["input_tokens"] + record["output_tokens"] + ) + buckets[bucket_key]["cost_usd"] += record["estimated_cost_usd"] + + # Round costs and sort by timestamp + result = [] + for bucket in buckets.values(): + bucket["cost_usd"] = round(bucket["cost_usd"], 6) + result.append(bucket) + + # Sort by timestamp + result.sort(key=lambda x: x["timestamp"]) + + return result + + def _get_bucket_key(self, timestamp: datetime, interval: str) -> str: + """Get the bucket key for a timestamp based on the interval. + + Args: + timestamp: Datetime to get bucket key for + interval: Time interval ('hour', 'day', 'week') + + Returns: + ISO 8601 formatted string representing the bucket start time + """ + if interval == "hour": + # Truncate to start of hour + bucket_start = timestamp.replace(minute=0, second=0, microsecond=0) + elif interval == "day": + # Truncate to start of day + bucket_start = timestamp.replace(hour=0, minute=0, second=0, microsecond=0) + elif interval == "week": + # Truncate to start of ISO week (Monday) + # Get the weekday (0=Monday, 6=Sunday) + days_since_monday = timestamp.weekday() + bucket_start = timestamp.replace(hour=0, minute=0, second=0, microsecond=0) + bucket_start = bucket_start - timedelta(days=days_since_monday) + else: + # This should never be reached due to validation in get_token_usage_timeseries + raise ValueError(f"Invalid interval: {interval}") + + # Return ISO format with Z suffix for UTC + return bucket_start.strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/codeframe/ui/routers/metrics.py b/codeframe/ui/routers/metrics.py index 52614f01..da1a9a40 100644 --- a/codeframe/ui/routers/metrics.py +++ b/codeframe/ui/routers/metrics.py @@ -10,7 +10,7 @@ """ import logging -from datetime import datetime +from datetime import datetime, timezone from typing import Optional from fastapi import APIRouter, Depends, HTTPException @@ -134,9 +134,147 @@ async def get_project_token_metrics( raise HTTPException(status_code=500, detail=f"Failed to retrieve token metrics: {str(e)}") +def _parse_date(date_str: str) -> datetime: + """Parse a date string in multiple formats. + + Accepts: + - ISO 8601 with time: '2025-01-01T00:00:00Z' + - Date only: '2025-01-01' (converts to start of day UTC) + + Args: + date_str: Date string to parse + + Returns: + Parsed datetime object + + Raises: + ValueError: If date format is invalid + """ + # Try full ISO 8601 format first + if "T" in date_str: + return datetime.fromisoformat(date_str.replace("Z", "+00:00")) + + # Try date-only format (yyyy-MM-dd) + try: + parsed = datetime.strptime(date_str, "%Y-%m-%d") + return parsed.replace(tzinfo=timezone.utc) + except ValueError: + raise ValueError( + f"Invalid date format: '{date_str}'. " + "Use ISO 8601 format (e.g., '2025-01-01T00:00:00Z' or '2025-01-01')" + ) + + +@router.get("/api/projects/{project_id}/metrics/tokens/timeseries") +async def get_project_token_timeseries( + project_id: int, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + interval: str = "day", + db: Database = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Get token usage time series data for charting. + + Returns token usage records aggregated by time intervals (hour, day, or week) + for visualization in charts. Each data point contains aggregated token counts + and costs for that time bucket. + + Args: + project_id: Project ID to get time series for + start_date: Start date (required). Accepts: + - ISO 8601: '2025-01-01T00:00:00Z' + - Date only: '2025-01-01' + end_date: End date (required). Same formats as start_date. + interval: Time interval for grouping ('hour', 'day', 'week'). Default: 'day' + db: Database instance (injected) + current_user: Authenticated user (injected) + + Returns: + 200 OK: Array of time series data points + [ + { + "timestamp": "2025-01-01T00:00:00Z", + "input_tokens": 1000, + "output_tokens": 500, + "total_tokens": 1500, + "cost_usd": 0.0105 + }, + ... + ] + 400 Bad Request: Missing required dates, invalid date format, or invalid interval + 404 Not Found: Project not found + 403 Forbidden: Access denied + 500 Internal Server Error: Processing error + + Example: + GET /api/projects/1/metrics/tokens/timeseries?start_date=2025-01-01&end_date=2025-01-07&interval=day + """ + # Validate required date parameters + if not start_date or not end_date: + raise HTTPException( + status_code=400, + detail="Both start_date and end_date are required for time series data", + ) + + # Validate project exists + project = db.get_project(project_id) + if not project: + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + + # Authorization check + if not db.user_has_project_access(current_user.id, project_id): + raise HTTPException(status_code=403, detail="Access denied") + + # Parse and validate date parameters + try: + start_dt = _parse_date(start_date) + end_dt = _parse_date(end_date) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + + # Validate interval + valid_intervals = ("hour", "day", "week") + if interval not in valid_intervals: + raise HTTPException( + status_code=400, + detail=f"Invalid interval '{interval}'. Must be one of: {', '.join(valid_intervals)}", + ) + + try: + # Get time series data using MetricsTracker + tracker = MetricsTracker(db=db) + timeseries = await tracker.get_token_usage_timeseries( + project_id=project_id, + start_date=start_dt, + end_date=end_dt, + interval=interval, + ) + + logger.info( + f"Retrieved time series for project {project_id}: " + f"{len(timeseries)} data points ({interval} interval)" + ) + + return timeseries + + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error( + f"Failed to get token time series for project {project_id}: {e}", + exc_info=True, + ) + raise HTTPException( + status_code=500, detail=f"Failed to retrieve token time series: {str(e)}" + ) + + @router.get("/api/projects/{project_id}/metrics/costs") async def get_project_cost_metrics( project_id: int, + start_date: Optional[str] = None, + end_date: Optional[str] = None, db: Database = Depends(get_db), current_user: User = Depends(get_current_user), ): @@ -146,10 +284,16 @@ async def get_project_cost_metrics( Returns total costs and breakdowns by agent and model for a project. Useful for understanding cost allocation and identifying high-cost operations. + Optionally filter by date range. Args: project_id: Project ID to get cost breakdown for + start_date: Optional start date. Accepts: + - ISO 8601: '2025-01-01T00:00:00Z' + - Date only: '2025-01-01' + end_date: Optional end date. Same formats as start_date. db: Database instance (injected) + current_user: Authenticated user (injected) Returns: 200 OK: Cost breakdown @@ -177,11 +321,14 @@ async def get_project_cost_metrics( ... ] } + 400 Bad Request: Invalid date format 404 Not Found: Project not found + 403 Forbidden: Access denied 500 Internal Server Error: Database or processing error Example: GET /api/projects/1/metrics/costs + GET /api/projects/1/metrics/costs?start_date=2025-01-01&end_date=2025-01-07 Response: { "project_id": 1, "total_cost_usd": 0.125, @@ -205,10 +352,23 @@ async def get_project_cost_metrics( if not db.user_has_project_access(current_user.id, project_id): raise HTTPException(status_code=403, detail="Access denied") + # Parse optional date parameters + start_dt = None + end_dt = None + try: + if start_date: + start_dt = _parse_date(start_date) + if end_date: + end_dt = _parse_date(end_date) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + try: # Get project costs using MetricsTracker tracker = MetricsTracker(db=db) - costs = await tracker.get_project_costs(project_id=project_id) + costs = await tracker.get_project_costs( + project_id=project_id, start_date=start_dt, end_date=end_dt + ) logger.info( f"Retrieved cost metrics for project {project_id}: ${costs['total_cost_usd']:.6f}" diff --git a/tests/api/test_api_metrics.py b/tests/api/test_api_metrics.py index c798a25c..a4008b8f 100644 --- a/tests/api/test_api_metrics.py +++ b/tests/api/test_api_metrics.py @@ -369,3 +369,261 @@ def test_all_endpoints_consistent(self, api_client, project_with_token_usage): assert backend_stats["cost_usd"] == agent_data["total_cost_usd"] assert backend_stats["total_tokens"] == agent_data["total_tokens"] assert backend_stats["call_count"] == agent_data["total_calls"] + + +class TestProjectTokenTimeSeriesEndpoint: + """Test GET /api/projects/{id}/metrics/tokens/timeseries endpoint.""" + + def test_endpoint_exists(self, api_client, project_with_token_usage): + """Test that endpoint exists and returns 200.""" + project_id, _ = project_with_token_usage + now = datetime.now(timezone.utc) + start_date = (now - timedelta(days=7)).strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries" + f"?start_date={start_date}&end_date={end_date}" + ) + assert response.status_code == 200 + + def test_returns_timeseries_structure(self, api_client, project_with_token_usage): + """Test that endpoint returns proper time series structure.""" + project_id, _ = project_with_token_usage + now = datetime.now(timezone.utc) + start_date = (now - timedelta(days=7)).strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries" + f"?start_date={start_date}&end_date={end_date}&interval=day" + ) + + assert response.status_code == 200 + data = response.json() + + # Response should be an array of time series data points + assert isinstance(data, list) + assert len(data) > 0 + + # Each data point should have the expected structure + for point in data: + assert "timestamp" in point + assert "input_tokens" in point + assert "output_tokens" in point + assert "total_tokens" in point + assert "cost_usd" in point + + def test_aggregates_by_day(self, api_client, project_with_token_usage): + """Test that data is properly aggregated by day interval.""" + project_id, _ = project_with_token_usage + now = datetime.now(timezone.utc) + start_date = (now - timedelta(days=3)).strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries" + f"?start_date={start_date}&end_date={end_date}&interval=day" + ) + + assert response.status_code == 200 + data = response.json() + + # Should have data points for days with usage + assert len(data) > 0 + + # Verify total tokens across all data points matches expected + total_input = sum(point["input_tokens"] for point in data) + total_output = sum(point["output_tokens"] for point in data) + + # We have 3 records: today (1500 tokens), yesterday (750), two days ago (3000) + # Filtering to last 3 days should include all + assert total_input + total_output > 0 + + def test_supports_hour_interval(self, api_client, project_with_token_usage): + """Test that hour interval is supported.""" + project_id, _ = project_with_token_usage + now = datetime.now(timezone.utc) + start_date = now.strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries" + f"?start_date={start_date}&end_date={end_date}&interval=hour" + ) + + assert response.status_code == 200 + + def test_supports_week_interval(self, api_client, project_with_token_usage): + """Test that week interval is supported.""" + project_id, _ = project_with_token_usage + now = datetime.now(timezone.utc) + start_date = (now - timedelta(days=14)).strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries" + f"?start_date={start_date}&end_date={end_date}&interval=week" + ) + + assert response.status_code == 200 + + def test_invalid_interval_returns_400(self, api_client, project_with_token_usage): + """Test that invalid interval returns 400 Bad Request.""" + project_id, _ = project_with_token_usage + now = datetime.now(timezone.utc) + start_date = (now - timedelta(days=7)).strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries" + f"?start_date={start_date}&end_date={end_date}&interval=invalid" + ) + + assert response.status_code == 400 + assert "interval" in response.json()["detail"].lower() + + def test_missing_dates_returns_400(self, api_client, project_with_token_usage): + """Test that missing required date parameters returns 400.""" + project_id, _ = project_with_token_usage + + # Missing start_date + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries?end_date=2025-01-01" + ) + assert response.status_code == 400 + + # Missing end_date + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries?start_date=2025-01-01" + ) + assert response.status_code == 400 + + def test_empty_date_range_returns_empty_array(self, api_client, project_with_token_usage): + """Test that date range with no data returns empty array.""" + project_id, _ = project_with_token_usage + + # Use date range far in the future with no data + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries" + f"?start_date=2099-01-01&end_date=2099-01-07" + ) + + assert response.status_code == 200 + data = response.json() + assert isinstance(data, list) + assert len(data) == 0 + + def test_nonexistent_project_returns_404(self, api_client): + """Test that nonexistent project returns 404.""" + response = api_client.get( + "/api/projects/99999/metrics/tokens/timeseries" + "?start_date=2025-01-01&end_date=2025-01-07" + ) + assert response.status_code == 404 + + def test_accepts_iso8601_dates(self, api_client, project_with_token_usage): + """Test that full ISO 8601 dates with time are accepted.""" + project_id, _ = project_with_token_usage + now = datetime.now(timezone.utc) + start_date = (now - timedelta(days=7)).isoformat().replace("+00:00", "Z") + end_date = now.isoformat().replace("+00:00", "Z") + + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries" + f"?start_date={start_date}&end_date={end_date}" + ) + + assert response.status_code == 200 + + def test_invalid_month_returns_400(self, api_client, project_with_token_usage): + """Test that invalid month (13) returns 400 Bad Request.""" + project_id, _ = project_with_token_usage + + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries" + f"?start_date=2025-13-01&end_date=2025-01-07" + ) + + assert response.status_code == 400 + assert "Invalid date format" in response.json()["detail"] + + def test_invalid_day_returns_400(self, api_client, project_with_token_usage): + """Test that invalid day (32) returns 400 Bad Request.""" + project_id, _ = project_with_token_usage + + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries" + f"?start_date=2025-01-32&end_date=2025-01-07" + ) + + assert response.status_code == 400 + assert "Invalid date format" in response.json()["detail"] + + def test_malformed_date_returns_400(self, api_client, project_with_token_usage): + """Test that malformed date string returns 400 Bad Request.""" + project_id, _ = project_with_token_usage + + response = api_client.get( + f"/api/projects/{project_id}/metrics/tokens/timeseries" + f"?start_date=not-a-date&end_date=2025-01-07" + ) + + assert response.status_code == 400 + assert "Invalid date format" in response.json()["detail"] + + +class TestProjectCostMetricsDateFiltering: + """Test date filtering on GET /api/projects/{id}/metrics/costs endpoint.""" + + def test_date_filtering_reduces_costs(self, api_client, project_with_token_usage): + """Test that date filtering reduces returned costs.""" + project_id, _ = project_with_token_usage + + # Get all-time costs + all_time_response = api_client.get(f"/api/projects/{project_id}/metrics/costs") + all_time_data = all_time_response.json() + + # Get today-only costs (should be less than all-time) + now = datetime.now(timezone.utc) + start_date = now.replace(hour=0, minute=0, second=0, microsecond=0).strftime("%Y-%m-%d") + end_date = now.strftime("%Y-%m-%d") + + filtered_response = api_client.get( + f"/api/projects/{project_id}/metrics/costs" + f"?start_date={start_date}&end_date={end_date}" + ) + + assert filtered_response.status_code == 200 + filtered_data = filtered_response.json() + + # Filtered should have fewer or equal costs (today only vs all time) + assert filtered_data["total_cost_usd"] <= all_time_data["total_cost_usd"] + assert filtered_data["total_calls"] <= all_time_data["total_calls"] + + def test_date_filtering_with_iso8601_format(self, api_client, project_with_token_usage): + """Test that ISO 8601 format works for costs date filtering.""" + project_id, _ = project_with_token_usage + now = datetime.now(timezone.utc) + start_date = (now - timedelta(days=7)).isoformat().replace("+00:00", "Z") + end_date = now.isoformat().replace("+00:00", "Z") + + response = api_client.get( + f"/api/projects/{project_id}/metrics/costs" + f"?start_date={start_date}&end_date={end_date}" + ) + + assert response.status_code == 200 + + def test_date_filtering_backwards_compatible(self, api_client, project_with_token_usage): + """Test that costs endpoint still works without date params (backward compatible).""" + project_id, _ = project_with_token_usage + + # Should work without any date params + response = api_client.get(f"/api/projects/{project_id}/metrics/costs") + assert response.status_code == 200 + + data = response.json() + assert "total_cost_usd" in data + assert "by_agent" in data + assert "by_model" in data