From 78150a4adcaf579da0e0c75510eb2b6d16ee74ed Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Fri, 26 Dec 2025 23:34:26 +0000 Subject: [PATCH 01/13] refactor: Clean up imports and improve code readability - Removed unused imports across multiple files, including asyncio and json. - Simplified conditional expressions for better clarity in lifespan and execution contexts. - Adjusted type hints and default values in models for consistency. - Enhanced formatting of long strings for improved readability in configuration files. --- src/api/dashboard_metrics.py | 2 +- src/api/exec.py | 5 -- src/api/files.py | 13 +--- src/api/health.py | 11 ++-- src/config/languages.py | 3 +- src/dependencies/services.py | 4 +- src/main.py | 8 +-- src/middleware/security.py | 4 +- src/models/api_key.py | 96 ++++++++++++++++++------------ src/models/errors.py | 2 +- src/models/exec.py | 5 +- src/models/execution.py | 2 +- src/models/metrics.py | 1 - src/models/session.py | 4 +- src/services/api_key_manager.py | 2 +- src/services/auth.py | 2 +- src/services/cleanup.py | 1 - src/services/container/executor.py | 1 - src/services/detailed_metrics.py | 7 +-- src/services/execution/runner.py | 8 ++- src/services/file.py | 2 - src/services/health.py | 6 +- src/services/interfaces.py | 1 - src/services/metrics.py | 2 +- src/services/orchestrator.py | 27 +++++---- src/services/session.py | 2 +- src/services/sqlite_metrics.py | 9 +-- src/services/state_archival.py | 1 - src/utils/config_validator.py | 1 - src/utils/error_handlers.py | 25 ++++---- 30 files changed, 124 insertions(+), 133 deletions(-) diff --git a/src/api/dashboard_metrics.py b/src/api/dashboard_metrics.py index 610b673..034852a 100644 --- a/src/api/dashboard_metrics.py +++ b/src/api/dashboard_metrics.py @@ -1,7 +1,7 @@ """Dashboard metrics API endpoints for advanced analytics.""" from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Literal, Optional +from typing import Dict, List, Literal, Optional from fastapi import APIRouter, Depends, Header, HTTPException, Query from pydantic import BaseModel diff --git a/src/api/exec.py b/src/api/exec.py index b148c05..62417af 100644 --- a/src/api/exec.py +++ b/src/api/exec.py @@ -9,11 +9,6 @@ from ..models import ExecRequest, ExecResponse from ..services.orchestrator import ExecutionOrchestrator -from ..services.interfaces import ( - SessionServiceInterface, - ExecutionServiceInterface, - FileServiceInterface, -) from ..dependencies.services import ( SessionServiceDep, FileServiceDep, diff --git a/src/api/files.py b/src/api/files.py index 7997963..7b38146 100644 --- a/src/api/files.py +++ b/src/api/files.py @@ -3,27 +3,18 @@ # Standard library imports from datetime import datetime, timezone from pathlib import Path -from typing import List, Optional, Union +from typing import List, Optional from urllib.parse import quote # Third-party imports import structlog -from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Form, Query +from fastapi import APIRouter, HTTPException, UploadFile, File, Form, Query from fastapi.responses import Response, StreamingResponse from unidecode import unidecode # Local application imports from ..config import settings from ..dependencies import FileServiceDep -from ..models import ( - FileUploadRequest, - FileUploadResponse, - FileInfo, - FileListResponse, - FileDownloadResponse, - FileDeleteResponse, -) -from ..services.interfaces import FileServiceInterface from ..utils.id_generator import generate_session_id diff --git a/src/api/health.py b/src/api/health.py index ce82304..79db3da 100644 --- a/src/api/health.py +++ b/src/api/health.py @@ -1,6 +1,5 @@ """Health check and monitoring endpoints.""" -from typing import Dict, Any from fastapi import APIRouter, HTTPException, Depends, Query from fastapi.responses import JSONResponse import structlog @@ -42,11 +41,11 @@ async def detailed_health_check( # Prepare response response_data = { "status": overall_status.value, - "timestamp": service_results[ - list(service_results.keys())[0] - ].timestamp.isoformat() - if service_results - else None, + "timestamp": ( + service_results[list(service_results.keys())[0]].timestamp.isoformat() + if service_results + else None + ), "services": { name: result.to_dict() for name, result in service_results.items() }, diff --git a/src/config/languages.py b/src/config/languages.py index 973c767..22715ce 100644 --- a/src/config/languages.py +++ b/src/config/languages.py @@ -59,7 +59,8 @@ class LanguageConfig: image="code-interpreter/nodejs:latest", user_id=1001, file_extension="ts", - execution_command="tsc /mnt/data/code.ts --outDir /mnt/data --module commonjs --target ES2019 && node /mnt/data/code.js", + execution_command="tsc /mnt/data/code.ts --outDir /mnt/data --module commonjs " + "--target ES2019 && node /mnt/data/code.js", uses_stdin=False, timeout_multiplier=1.2, memory_multiplier=1.0, diff --git a/src/dependencies/services.py b/src/dependencies/services.py index 8eabcbb..ce21af2 100644 --- a/src/dependencies/services.py +++ b/src/dependencies/services.py @@ -2,10 +2,10 @@ # Standard library imports from functools import lru_cache -from typing import Annotated, Optional +from typing import Annotated # Third-party imports -from fastapi import Depends, Request +from fastapi import Depends import structlog # Local application imports diff --git a/src/main.py b/src/main.py index c6e74c1..0fab6d6 100644 --- a/src/main.py +++ b/src/main.py @@ -1,8 +1,6 @@ """Main FastAPI application for the Code Interpreter API.""" # Standard library imports -import asyncio -import os import sys from contextlib import asynccontextmanager @@ -121,9 +119,9 @@ async def lifespan(app: FastAPI): cleanup_scheduler.set_services( execution_service=get_execution_service(), file_service=get_file_service(), - state_archival_service=get_state_archival_service() - if settings.state_archive_enabled - else None, + state_archival_service=( + get_state_archival_service() if settings.state_archive_enabled else None + ), ) cleanup_scheduler.start() logger.info( diff --git a/src/middleware/security.py b/src/middleware/security.py index 2a3ae92..6cc4539 100644 --- a/src/middleware/security.py +++ b/src/middleware/security.py @@ -2,7 +2,7 @@ # Standard library imports import time -from typing import Callable, Dict, Any, Optional +from typing import Callable, Optional # Third-party imports import structlog @@ -152,8 +152,6 @@ def _should_skip_auth(self, request: Request) -> bool: async def _authenticate_request(self, request: Request, scope: dict): """Handle API key authentication with rate limiting.""" - import hashlib - # Extract API key api_key = self._extract_api_key(request) diff --git a/src/models/api_key.py b/src/models/api_key.py index 703008b..e4d7320 100644 --- a/src/models/api_key.py +++ b/src/models/api_key.py @@ -2,7 +2,7 @@ import json from dataclasses import dataclass, field -from datetime import datetime, timezone +from datetime import datetime from typing import Optional, Dict, Any @@ -76,21 +76,31 @@ def to_redis_hash(self) -> Dict[str, str]: "name": self.name, "created_at": self.created_at.isoformat(), "enabled": "true" if self.enabled else "false", - "rate_limits_per_second": str(self.rate_limits.per_second) - if self.rate_limits.per_second is not None - else "", - "rate_limits_per_minute": str(self.rate_limits.per_minute) - if self.rate_limits.per_minute is not None - else "", - "rate_limits_hourly": str(self.rate_limits.hourly) - if self.rate_limits.hourly is not None - else "", - "rate_limits_daily": str(self.rate_limits.daily) - if self.rate_limits.daily is not None - else "", - "rate_limits_monthly": str(self.rate_limits.monthly) - if self.rate_limits.monthly is not None - else "", + "rate_limits_per_second": ( + str(self.rate_limits.per_second) + if self.rate_limits.per_second is not None + else "" + ), + "rate_limits_per_minute": ( + str(self.rate_limits.per_minute) + if self.rate_limits.per_minute is not None + else "" + ), + "rate_limits_hourly": ( + str(self.rate_limits.hourly) + if self.rate_limits.hourly is not None + else "" + ), + "rate_limits_daily": ( + str(self.rate_limits.daily) + if self.rate_limits.daily is not None + else "" + ), + "rate_limits_monthly": ( + str(self.rate_limits.monthly) + if self.rate_limits.monthly is not None + else "" + ), "metadata": json.dumps(self.metadata), "last_used_at": self.last_used_at.isoformat() if self.last_used_at else "", "usage_count": str(self.usage_count), @@ -101,31 +111,39 @@ def from_redis_hash(cls, data: Dict[bytes, bytes]) -> "ApiKeyRecord": """Create from Redis hash data (bytes keys/values).""" # Decode bytes to strings decoded = { - k.decode() - if isinstance(k, bytes) - else k: v.decode() - if isinstance(v, bytes) - else v + k.decode() if isinstance(k, bytes) else k: ( + v.decode() if isinstance(v, bytes) else v + ) for k, v in data.items() } # Parse rate limits rate_limits = RateLimits( - per_second=int(decoded["rate_limits_per_second"]) - if decoded.get("rate_limits_per_second") - else None, - per_minute=int(decoded["rate_limits_per_minute"]) - if decoded.get("rate_limits_per_minute") - else None, - hourly=int(decoded["rate_limits_hourly"]) - if decoded.get("rate_limits_hourly") - else None, - daily=int(decoded["rate_limits_daily"]) - if decoded.get("rate_limits_daily") - else None, - monthly=int(decoded["rate_limits_monthly"]) - if decoded.get("rate_limits_monthly") - else None, + per_second=( + int(decoded["rate_limits_per_second"]) + if decoded.get("rate_limits_per_second") + else None + ), + per_minute=( + int(decoded["rate_limits_per_minute"]) + if decoded.get("rate_limits_per_minute") + else None + ), + hourly=( + int(decoded["rate_limits_hourly"]) + if decoded.get("rate_limits_hourly") + else None + ), + daily=( + int(decoded["rate_limits_daily"]) + if decoded.get("rate_limits_daily") + else None + ), + monthly=( + int(decoded["rate_limits_monthly"]) + if decoded.get("rate_limits_monthly") + else None + ), ) # Parse timestamps @@ -158,9 +176,9 @@ def to_display_dict(self) -> Dict[str, Any]: "name": self.name, "enabled": self.enabled, "created_at": self.created_at.isoformat(), - "last_used_at": self.last_used_at.isoformat() - if self.last_used_at - else None, + "last_used_at": ( + self.last_used_at.isoformat() if self.last_used_at else None + ), "usage_count": self.usage_count, "rate_limits": { "hourly": self.rate_limits.hourly, diff --git a/src/models/errors.py b/src/models/errors.py index 29aa941..b64f2b9 100644 --- a/src/models/errors.py +++ b/src/models/errors.py @@ -1,7 +1,7 @@ """Error models and exception classes for the Code Interpreter API.""" import time -from typing import Optional, Dict, Any, List +from typing import Optional, List from pydantic import BaseModel, Field from enum import Enum diff --git a/src/models/exec.py b/src/models/exec.py index 35d0682..94572ea 100644 --- a/src/models/exec.py +++ b/src/models/exec.py @@ -2,8 +2,7 @@ # Standard library imports from datetime import datetime -from enum import Enum -from typing import List, Optional, Dict, Any +from typing import List, Optional, Any # Third-party imports from pydantic import BaseModel, Field @@ -45,7 +44,7 @@ class ExecRequest(BaseModel): default=None, description="Optional session ID to continue an existing session (for state persistence)", ) - files: Optional[List[RequestFile]] = Field( + files: List[RequestFile] = Field( default_factory=list, description="Array of file references to be used during execution", ) diff --git a/src/models/execution.py b/src/models/execution.py index fddea69..5a0d45d 100644 --- a/src/models/execution.py +++ b/src/models/execution.py @@ -3,7 +3,7 @@ # Standard library imports from datetime import datetime from enum import Enum -from typing import List, Optional, Dict, Any +from typing import List, Optional # Third-party imports from pydantic import BaseModel, Field diff --git a/src/models/metrics.py b/src/models/metrics.py index 9fef179..4a0ef70 100644 --- a/src/models/metrics.py +++ b/src/models/metrics.py @@ -7,7 +7,6 @@ - Detailed resource consumption """ -import json from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Dict, Any, Optional, List diff --git a/src/models/session.py b/src/models/session.py index 5e89c8a..58e9000 100644 --- a/src/models/session.py +++ b/src/models/session.py @@ -3,7 +3,7 @@ # Standard library imports from datetime import datetime from enum import Enum -from typing import Dict, List, Optional, Any +from typing import Dict, Optional, Any # Third-party imports from pydantic import BaseModel, Field @@ -77,7 +77,7 @@ class Config: class SessionCreate(BaseModel): """Request model for creating a new session.""" - metadata: Optional[Dict[str, Any]] = Field( + metadata: Dict[str, Any] = Field( default_factory=dict, description="Optional session metadata" ) diff --git a/src/services/api_key_manager.py b/src/services/api_key_manager.py index 6aff1fd..03e0cf2 100644 --- a/src/services/api_key_manager.py +++ b/src/services/api_key_manager.py @@ -11,7 +11,7 @@ import hmac import secrets from datetime import datetime, timezone, timedelta -from typing import Optional, Tuple, List, Dict, Any +from typing import Optional, Tuple, List, Dict import redis.asyncio as redis import structlog diff --git a/src/services/auth.py b/src/services/auth.py index 0180e46..ad013aa 100644 --- a/src/services/auth.py +++ b/src/services/auth.py @@ -9,7 +9,7 @@ import hashlib import hmac from datetime import datetime -from typing import Optional, Dict, Any, Tuple +from typing import Optional, Dict, Any # Third-party imports import redis.asyncio as redis diff --git a/src/services/cleanup.py b/src/services/cleanup.py index de06be1..2444a69 100644 --- a/src/services/cleanup.py +++ b/src/services/cleanup.py @@ -12,7 +12,6 @@ import asyncio from typing import Dict, Set, Optional -from datetime import datetime import structlog diff --git a/src/services/container/executor.py b/src/services/container/executor.py index b82305b..8bf563d 100644 --- a/src/services/container/executor.py +++ b/src/services/container/executor.py @@ -3,7 +3,6 @@ import asyncio import re import shlex -from datetime import datetime from typing import Any, Dict, Optional, Tuple import structlog diff --git a/src/services/detailed_metrics.py b/src/services/detailed_metrics.py index ed4f1e1..97f3e74 100644 --- a/src/services/detailed_metrics.py +++ b/src/services/detailed_metrics.py @@ -8,7 +8,6 @@ """ import json -from collections import defaultdict from datetime import datetime, timezone, timedelta from typing import Optional, List, Dict, Any @@ -129,9 +128,9 @@ async def record_execution(self, metrics: DetailedExecutionMetrics) -> None: "Recorded detailed metrics", execution_id=metrics.execution_id, language=metrics.language, - api_key_hash=metrics.api_key_hash[:8] - if metrics.api_key_hash - else "unknown", + api_key_hash=( + metrics.api_key_hash[:8] if metrics.api_key_hash else "unknown" + ), ) except Exception as e: diff --git a/src/services/execution/runner.py b/src/services/execution/runner.py index b654cdb..e98f8f4 100644 --- a/src/services/execution/runner.py +++ b/src/services/execution/runner.py @@ -370,9 +370,11 @@ def _record_metrics( memory_peak_mb=execution.memory_peak_mb, exit_code=execution.exit_code, file_count=len(files) if files else 0, - output_size_bytes=sum(len(o.content) for o in execution.outputs) - if execution.outputs - else 0, + output_size_bytes=( + sum(len(o.content) for o in execution.outputs) + if execution.outputs + else 0 + ), ) metrics_collector.record_execution_metrics(metrics) except Exception as e: diff --git a/src/services/file.py b/src/services/file.py index d2793fb..0eda01d 100644 --- a/src/services/file.py +++ b/src/services/file.py @@ -2,10 +2,8 @@ # Standard library imports import asyncio -import json from datetime import datetime, timedelta from typing import List, Optional, Tuple, Dict, Any -from urllib.parse import urljoin # Third-party imports import redis.asyncio as redis diff --git a/src/services/health.py b/src/services/health.py index 6c6d409..b09ec3c 100644 --- a/src/services/health.py +++ b/src/services/health.py @@ -5,7 +5,7 @@ import time from datetime import datetime, timezone from enum import Enum -from typing import Dict, Any, Optional, List +from typing import Dict, Any, Optional # Third-party imports import docker @@ -404,9 +404,7 @@ async def check_docker(self) -> HealthCheckResult: "running_containers": running_containers, "registry_accessible": registry_accessible, "server_version": system_info.get("ServerVersion", "unknown"), - "memory_total_gb": round( - system_info.get("MemTotal", 0) / (1024**3), 2 - ), + "memory_total_gb": round(system_info.get("MemTotal", 0) / (1024**3), 2), "cpu_count": system_info.get("NCPU", 0), } diff --git a/src/services/interfaces.py b/src/services/interfaces.py index 88521aa..ab50899 100644 --- a/src/services/interfaces.py +++ b/src/services/interfaces.py @@ -2,7 +2,6 @@ # Standard library imports from abc import ABC, abstractmethod -from datetime import datetime from typing import List, Optional, Dict, Any, Tuple # Local application imports diff --git a/src/services/metrics.py b/src/services/metrics.py index 4659297..a766244 100644 --- a/src/services/metrics.py +++ b/src/services/metrics.py @@ -5,7 +5,7 @@ import time from collections import defaultdict, deque from dataclasses import dataclass, field -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone from enum import Enum from typing import Dict, Any, Optional, List diff --git a/src/services/orchestrator.py b/src/services/orchestrator.py index 4d7c04e..f58614d 100644 --- a/src/services/orchestrator.py +++ b/src/services/orchestrator.py @@ -30,7 +30,6 @@ ExecRequest, ExecResponse, FileRef, - RequestFile, SessionCreate, ExecuteCodeRequest, ValidationError, @@ -63,9 +62,9 @@ class ExecutionContext: generated_files: Optional[List[FileRef]] = None stdout: str = "" stderr: str = "" - container: Optional[ - Any - ] = None # Container used for execution (avoids session lookup) + container: Optional[Any] = ( + None # Container used for execution (avoids session lookup) + ) # State persistence fields initial_state: Optional[str] = None new_state: Optional[str] = None @@ -467,9 +466,11 @@ async def _execute_code(self, ctx: ExecutionContext) -> Any: "Code execution completed", session_id=ctx.session_id, status=execution.status.value, - container_id=ctx.container.id[:12] - if ctx.container and hasattr(ctx.container, "id") - else None, + container_id=( + ctx.container.id[:12] + if ctx.container and hasattr(ctx.container, "id") + else None + ), has_state=ctx.new_state is not None, ) @@ -649,9 +650,9 @@ async def destroy_background(): await event_bus.publish( ExecutionCompleted( - execution_id=ctx.execution.execution_id - if ctx.execution - else ctx.request_id, + execution_id=( + ctx.execution.execution_id if ctx.execution else ctx.request_id + ), session_id=ctx.session_id, success=success, execution_time_ms=execution_time_ms, @@ -707,9 +708,9 @@ async def _record_detailed_metrics( ) metrics = DetailedExecutionMetrics( - execution_id=ctx.execution.execution_id - if ctx.execution - else ctx.request_id, + execution_id=( + ctx.execution.execution_id if ctx.execution else ctx.request_id + ), session_id=ctx.session_id or "", api_key_hash=ctx.api_key_hash[:16] if ctx.api_key_hash else "unknown", user_id=ctx.request.user_id, diff --git a/src/services/session.py b/src/services/session.py index f405ca3..162e1d6 100644 --- a/src/services/session.py +++ b/src/services/session.py @@ -4,7 +4,7 @@ import asyncio import json from datetime import datetime, timedelta, timezone -from typing import List, Optional, Dict, Any +from typing import List, Optional, Dict # Third-party imports import redis.asyncio as redis diff --git a/src/services/sqlite_metrics.py b/src/services/sqlite_metrics.py index dc5b863..dc80372 100644 --- a/src/services/sqlite_metrics.py +++ b/src/services/sqlite_metrics.py @@ -5,7 +5,6 @@ """ import asyncio -import os from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Dict, List, Optional @@ -230,9 +229,11 @@ async def _write_batch(self, batch: List[DetailedExecutionMetrics]) -> None: m.files_generated, m.output_size_bytes, m.state_size_bytes, - m.timestamp.isoformat() - if m.timestamp - else datetime.now(timezone.utc).isoformat(), + ( + m.timestamp.isoformat() + if m.timestamp + else datetime.now(timezone.utc).isoformat() + ), ) for m in batch ], diff --git a/src/services/state_archival.py b/src/services/state_archival.py index 5b8b1fc..ed70f4c 100644 --- a/src/services/state_archival.py +++ b/src/services/state_archival.py @@ -19,7 +19,6 @@ import asyncio import io -import json from datetime import datetime, timezone from typing import Optional, Dict, Any, List diff --git a/src/utils/config_validator.py b/src/utils/config_validator.py index a794918..5a9c0fd 100644 --- a/src/utils/config_validator.py +++ b/src/utils/config_validator.py @@ -2,7 +2,6 @@ import logging from typing import List, Dict, Any -from pathlib import Path import docker import redis from minio import Minio diff --git a/src/utils/error_handlers.py b/src/utils/error_handlers.py index 305f0db..7902f1b 100644 --- a/src/utils/error_handlers.py +++ b/src/utils/error_handlers.py @@ -2,7 +2,6 @@ # Standard library imports import traceback -import uuid from typing import Union # Third-party imports @@ -50,9 +49,9 @@ async def code_interpreter_exception_handler( "request_id": exc.request_id, "path": request.url.path, "method": request.method, - "client_ip": getattr(request.client, "host", "unknown") - if request.client - else "unknown", + "client_ip": ( + getattr(request.client, "host", "unknown") if request.client else "unknown" + ), } # Add details if present @@ -109,9 +108,9 @@ async def http_exception_handler(request: Request, exc: HTTPException) -> JSONRe request_id=request_id, path=request.url.path, method=request.method, - client_ip=getattr(request.client, "host", "unknown") - if request.client - else "unknown", + client_ip=( + getattr(request.client, "host", "unknown") if request.client else "unknown" + ), ) # Create standardized error response @@ -148,9 +147,9 @@ async def validation_exception_handler( validation_errors=[ {"field": d.field, "message": d.message, "code": d.code} for d in details ], - client_ip=getattr(request.client, "host", "unknown") - if request.client - else "unknown", + client_ip=( + getattr(request.client, "host", "unknown") if request.client else "unknown" + ), ) # Create standardized error response @@ -178,9 +177,9 @@ async def general_exception_handler(request: Request, exc: Exception) -> JSONRes exception_type=type(exc).__name__, exception_message=str(exc), traceback=traceback.format_exc(), - client_ip=getattr(request.client, "host", "unknown") - if request.client - else "unknown", + client_ip=( + getattr(request.client, "host", "unknown") if request.client else "unknown" + ), ) # Create generic error response (don't expose internal details) From 23a168f9bf0f37cbe7b3b9d0ea4d712c3784e028 Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 00:02:32 +0000 Subject: [PATCH 02/13] chore: Update .env.example to reflect CPU configuration changes - Replaced MAX_CPU with MAX_CPUS for clarity. - Marked MAX_CPU_QUOTA as deprecated for future reference. --- .env.example | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 463c1a3..3c914a4 100644 --- a/.env.example +++ b/.env.example @@ -52,7 +52,8 @@ DOCKER_READ_ONLY=true # Resource Limits - Execution MAX_EXECUTION_TIME=30 MAX_MEMORY_MB=512 -MAX_CPU_QUOTA=50000 +MAX_CPUS=1 +MAX_CPU_QUOTA=50000 #Deprecated MAX_PROCESSES=32 MAX_OPEN_FILES=1024 From 2929962d62f57cc6f91850123ffedcb42f2af419 Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 00:37:19 +0000 Subject: [PATCH 03/13] docs: Update README to reflect changes in execution environment setup - Changed instructions from pulling prebuilt Docker images to building the execution environment. - Updated commands for building Python and all language images accordingly. --- README.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 3c0f1a8..35dcb8c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ A secure, open-source code interpreter API that provides sandboxed code executio ## Quick Start -Get up and running in minutes using prebuilt Docker images. +Get up and running in minutes by building the execution environment. 1. **Clone the repository** @@ -24,16 +24,14 @@ Get up and running in minutes using prebuilt Docker images. # The default settings work out-of-the-box for local development ``` -3. **Pull execution environment images** +3. **Build execution environment images** ```bash - # Python only (minimal) - docker pull ghcr.io/usnavy13/librecodeinterpreter/python:latest + # Build Python only (minimal) + ./docker/build-images.sh -l python - # Or pull all 12 languages - for lang in python nodejs go java c-cpp php rust r fortran d; do - docker pull ghcr.io/usnavy13/librecodeinterpreter/$lang:latest - done + # Or build all 12 languages + ./docker/build-images.sh -p ``` 4. **Start the API** @@ -41,8 +39,6 @@ Get up and running in minutes using prebuilt Docker images. docker compose up -d ``` -> **Building from source?** See the [Development Guide](docs/DEVELOPMENT.md) for instructions on building images locally. - The API will be available at `http://localhost:8000`. Visit `http://localhost:8000/docs` for the interactive API documentation. From 1ed81c5f7864b3656078d036827b350dfd0d0343 Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 01:15:29 +0000 Subject: [PATCH 04/13] feat: Add Docker Compose configuration and update environment settings - Introduced a new `docker-compose.ghcr.yml` file for deploying the application with pre-built images from GitHub Container Registry. - Updated `.env.example` to include `DOCKER_IMAGE_REGISTRY` for better configuration management. - Enhanced the `Settings` class to support dynamic Docker image retrieval based on the specified registry. - Modified `languages.py` to use images without the registry prefix for better compatibility with the new setup. - Updated the README to reflect changes in execution environment setup and provide instructions for using local vs. pre-built images. --- .env.example | 1 + README.md | 24 ++++- docker-compose.ghcr.yml | 156 ++++++++++++++++++++++++++++++ src/config/__init__.py | 29 +++++- src/config/languages.py | 28 +++--- src/services/container/manager.py | 3 +- 6 files changed, 220 insertions(+), 21 deletions(-) create mode 100644 docker-compose.ghcr.yml diff --git a/.env.example b/.env.example index 3c914a4..f87fe23 100644 --- a/.env.example +++ b/.env.example @@ -44,6 +44,7 @@ MINIO_BUCKET=code-interpreter-files MINIO_REGION=us-east-1 # Docker Configuration +DOCKER_IMAGE_REGISTRY=code-interpreter # DOCKER_BASE_URL=unix://var/run/docker.sock DOCKER_TIMEOUT=60 DOCKER_NETWORK_MODE=none diff --git a/README.md b/README.md index 35dcb8c..e017f69 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,11 @@ Get up and running in minutes by building the execution environment. # The default settings work out-of-the-box for local development ``` -3. **Build execution environment images** +3. **Prepare execution environment images** + You can either build the images locally (recommended) or pull pre-built images from GitHub Container Registry. + + **Option A: Build locally (Recommended)** ```bash # Build Python only (minimal) ./docker/build-images.sh -l python @@ -34,11 +37,30 @@ Get up and running in minutes by building the execution environment. ./docker/build-images.sh -p ``` + **Option B: Pull from GHCR** + ```bash + # Pull Python only + docker pull ghcr.io/usnavy13/librecodeinterpreter/python:latest + + # Or pull the API and all languages + docker pull ghcr.io/usnavy13/librecodeinterpreter:latest + for lang in python nodejs go java c-cpp php rust r fortran d; do + docker pull ghcr.io/usnavy13/librecodeinterpreter/$lang:latest + done + ``` + 4. **Start the API** + + **Option A: Using local images (if you built them)** ```bash docker compose up -d ``` + **Option B: Using pre-built images (if you pulled them)** + ```bash + docker compose -f docker-compose.ghcr.yml up -d + ``` + The API will be available at `http://localhost:8000`. Visit `http://localhost:8000/docs` for the interactive API documentation. diff --git a/docker-compose.ghcr.yml b/docker-compose.ghcr.yml new file mode 100644 index 0000000..7e9a7d8 --- /dev/null +++ b/docker-compose.ghcr.yml @@ -0,0 +1,156 @@ +services: + # Code Interpreter API + api: + image: ghcr.io/usnavy13/librecodeinterpreter:latest + container_name: code-interpreter-api + user: "1000:988" # Run as user with docker group access + cap_add: + - NET_ADMIN # Required for iptables management when WAN access is enabled + ports: + - "${API_PORT:-8000}:8000" + - "${HTTPS_PORT:-443}:443" + env_file: + - .env + environment: + # Container-specific overrides (these override .env values) + - API_HOST=0.0.0.0 + - API_PORT=8000 + - DOCKER_IMAGE_REGISTRY=ghcr.io/usnavy13/librecodeinterpreter + + # Service discovery (container names) + - REDIS_HOST=redis + - REDIS_PORT=6379 + - MINIO_ENDPOINT=minio:9000 + + # Docker socket path inside container + - DOCKER_BASE_URL=unix://var/run/docker.sock + + # SSL paths inside container + - SSL_CERT_FILE=/app/ssl/cert.pem + - SSL_KEY_FILE=/app/ssl/key.pem + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./logs:/app/logs + - ./data:/app/data + - ./ssl:/app/ssl + depends_on: + - redis + - minio + networks: + - code-interpreter-network + restart: unless-stopped + stop_grace_period: 30s # Allow time to cleanup pooled containers + healthcheck: + test: + [ + "CMD", + "sh", + "-c", + "curl -f -k https://localhost:443/health 2>/dev/null || curl -f http://localhost:8000/health 2>/dev/null || curl -f http://localhost:80/health || exit 1", + ] + interval: 30s + timeout: 15s + retries: 3 + start_period: 15s + + # Redis for session management + redis: + image: redis:7-alpine + container_name: code-interpreter-redis + ports: + # Expose to localhost for CLI tools + - "127.0.0.1:6379:6379" + environment: + - REDIS_PASSWORD=${REDIS_PASSWORD:-} + command: > + sh -c " + if [ -n \"$$REDIS_PASSWORD\" ]; then + redis-server --requirepass $$REDIS_PASSWORD --appendonly yes --appendfsync everysec + else + redis-server --appendonly yes --appendfsync everysec + fi + " + volumes: + - redis-data:/data + - ./docker/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro + networks: + - code-interpreter-network + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 30s + timeout: 10s + retries: 3 + + # MinIO for file storage + minio: + image: minio/minio:latest + container_name: code-interpreter-minio + ports: + # API port for local testing + - "127.0.0.1:9000:9000" + # Console only, bound to localhost (access via SSH tunnel) + - "127.0.0.1:${MINIO_CONSOLE_PORT:-9001}:9001" + environment: + - MINIO_ROOT_USER=${MINIO_ACCESS_KEY:-minioadmin} + - MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY:-minioadmin} + - MINIO_BROWSER_REDIRECT_URL=http://localhost:9001 + command: server /data --console-address ":9001" + volumes: + - minio-data:/data + networks: + - code-interpreter-network + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 10s + retries: 3 + + # MinIO bucket initialization + minio-init: + image: minio/mc:latest + container_name: code-interpreter-minio-init + depends_on: + - minio + environment: + - MINIO_ENDPOINT=minio:9000 + - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-minioadmin} + - MINIO_SECRET_KEY=${MINIO_SECRET_KEY:-minioadmin} + - MINIO_BUCKET=${MINIO_BUCKET:-code-interpreter-files} + entrypoint: > + /bin/sh -c " echo 'Waiting for MinIO to be ready...'; until mc alias set minio http://$$MINIO_ENDPOINT $$MINIO_ACCESS_KEY $$MINIO_SECRET_KEY; do + echo 'MinIO not ready, waiting...'; + sleep 2; + done; echo 'MinIO is ready. Creating bucket if it does not exist...'; mc mb minio/$$MINIO_BUCKET --ignore-existing; echo 'Bucket setup complete.'; " + networks: + - code-interpreter-network + +volumes: + redis-data: + driver: local + minio-data: + driver: local + +networks: + code-interpreter-network: + driver: bridge + ipam: + config: + - subnet: 172.20.0.0/16 + + # WAN-only network for execution containers that need internet access + # This network is only created when ENABLE_WAN_ACCESS=true + code-interpreter-wan: + driver: bridge + ipam: + config: + - subnet: 172.30.0.0/16 + driver_opts: + # Enable NAT for outbound internet access + com.docker.network.bridge.enable_ip_masquerade: "true" + # Disable inter-container communication + com.docker.network.bridge.enable_icc: "false" + labels: + com.code-interpreter.managed: "true" + com.code-interpreter.type: "wan-access" diff --git a/src/config/__init__.py b/src/config/__init__.py index 5faeb50..d0b15e4 100644 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -111,6 +111,10 @@ class Settings(BaseSettings): # Docker Configuration docker_base_url: Optional[str] = Field(default=None) + docker_image_registry: str = Field( + default="code-interpreter", + description="Registry/namespace prefix for execution environment images", + ) docker_timeout: int = Field(default=60, ge=10) docker_network_mode: str = Field(default="none") docker_security_opt: List[str] = Field( @@ -384,16 +388,23 @@ class Settings(BaseSettings): enable_filesystem_isolation: bool = Field(default=True) # Language Configuration - now uses LANGUAGES from languages.py - supported_languages: Dict[str, Dict[str, Any]] = Field( - default_factory=lambda: { + supported_languages: Dict[str, Dict[str, Any]] = Field(default_factory=dict) + + @validator("supported_languages", pre=True, always=True) + def _set_supported_languages(cls, v, values): + """Initialize supported_languages with registry-prefixed images.""" + if v: + return v + + registry = values.get("docker_image_registry", "code-interpreter") + return { code: { - "image": lang.image, + "image": f"{registry}/{lang.image}" if registry else lang.image, "timeout_multiplier": lang.timeout_multiplier, "memory_multiplier": lang.memory_multiplier, } for code, lang in LANGUAGES.items() } - ) # Logging Configuration log_level: str = Field(default="INFO") @@ -580,6 +591,16 @@ def get_language_config(self, language: str) -> Dict[str, Any]: """Get configuration for a specific language.""" return self.supported_languages.get(language, {}) + def get_image_for_language(self, code: str) -> str: + """Get Docker image for a language.""" + config = self.get_language_config(code) + if config and "image" in config: + return config["image"] + + # Fallback to languages.py logic if not in settings + from .languages import get_image_for_language as get_img + return get_img(code, registry=self.docker_image_registry) + def get_execution_timeout(self, language: str) -> int: """Get execution timeout for a specific language.""" multiplier = self.get_language_config(language).get("timeout_multiplier", 1.0) diff --git a/src/config/languages.py b/src/config/languages.py index 22715ce..a00a4d4 100644 --- a/src/config/languages.py +++ b/src/config/languages.py @@ -34,7 +34,7 @@ class LanguageConfig: "py": LanguageConfig( code="py", name="Python", - image="code-interpreter/python:latest", + image="python:latest", user_id=999, file_extension="py", execution_command="python3 -", @@ -45,7 +45,7 @@ class LanguageConfig: "js": LanguageConfig( code="js", name="JavaScript", - image="code-interpreter/nodejs:latest", + image="nodejs:latest", user_id=1001, file_extension="js", execution_command="node", @@ -56,7 +56,7 @@ class LanguageConfig: "ts": LanguageConfig( code="ts", name="TypeScript", - image="code-interpreter/nodejs:latest", + image="nodejs:latest", user_id=1001, file_extension="ts", execution_command="tsc /mnt/data/code.ts --outDir /mnt/data --module commonjs " @@ -68,7 +68,7 @@ class LanguageConfig: "go": LanguageConfig( code="go", name="Go", - image="code-interpreter/go:latest", + image="go:latest", user_id=1001, file_extension="go", execution_command="go build -o code code.go && ./code", @@ -79,7 +79,7 @@ class LanguageConfig: "java": LanguageConfig( code="java", name="Java", - image="code-interpreter/java:latest", + image="java:latest", user_id=999, file_extension="java", execution_command="javac Code.java && java Code", @@ -90,7 +90,7 @@ class LanguageConfig: "c": LanguageConfig( code="c", name="C", - image="code-interpreter/c-cpp:latest", + image="c-cpp:latest", user_id=1001, file_extension="c", execution_command="gcc -o code code.c && ./code", @@ -101,7 +101,7 @@ class LanguageConfig: "cpp": LanguageConfig( code="cpp", name="C++", - image="code-interpreter/c-cpp:latest", + image="c-cpp:latest", user_id=1001, file_extension="cpp", execution_command="g++ -o code code.cpp && ./code", @@ -112,7 +112,7 @@ class LanguageConfig: "php": LanguageConfig( code="php", name="PHP", - image="code-interpreter/php:latest", + image="php:latest", user_id=1001, file_extension="php", execution_command="php", @@ -123,7 +123,7 @@ class LanguageConfig: "rs": LanguageConfig( code="rs", name="Rust", - image="code-interpreter/rust:latest", + image="rust:latest", user_id=1001, file_extension="rs", execution_command="rustc code.rs -o code && ./code", @@ -134,7 +134,7 @@ class LanguageConfig: "r": LanguageConfig( code="r", name="R", - image="code-interpreter/r:latest", + image="r:latest", user_id=1001, file_extension="r", execution_command="Rscript /dev/stdin", @@ -145,7 +145,7 @@ class LanguageConfig: "f90": LanguageConfig( code="f90", name="Fortran", - image="code-interpreter/fortran:latest", + image="fortran:latest", user_id=1001, file_extension="f90", execution_command="gfortran -o code code.f90 && ./code", @@ -156,7 +156,7 @@ class LanguageConfig: "d": LanguageConfig( code="d", name="D", - image="code-interpreter/d:latest", + image="d:latest", user_id=0, file_extension="d", execution_command="ldc2 code.d -of=code && ./code", @@ -183,11 +183,11 @@ def is_supported_language(code: str) -> bool: # Convenience lookups for backward compatibility during transition -def get_image_for_language(code: str) -> str: +def get_image_for_language(code: str, registry: Optional[str] = None) -> str: """Get Docker image for a language.""" lang = get_language(code) if lang: - return lang.image + return f"{registry}/{lang.image}" if registry else lang.image raise ValueError(f"Unsupported language: {code}") diff --git a/src/services/container/manager.py b/src/services/container/manager.py index 14664fb..8391eb7 100644 --- a/src/services/container/manager.py +++ b/src/services/container/manager.py @@ -13,7 +13,6 @@ from ...config import settings from ...config.languages import ( - get_image_for_language, get_user_id_for_language, ) from .client import DockerClientFactory @@ -57,7 +56,7 @@ def reset_initialization(self) -> None: def get_image_for_language(self, language: str) -> str: """Get Docker image for a programming language.""" - return get_image_for_language(language.lower().strip()) + return settings.get_image_for_language(language.lower().strip()) def get_user_id_for_language(self, language: str) -> int: """Get the user ID for a language container.""" From 83322eb9a6ebe5f4adf644a3bd0e87e6f74e0ae0 Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 01:44:17 +0000 Subject: [PATCH 05/13] feat: Add multi-platform support for Docker builds - Updated the Docker publish workflows to support multi-platform builds for linux/amd64 and linux/arm64 architectures. --- .github/workflows/docker-publish.yml | 1 + .github/workflows/execution-env-publish.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 616b5d7..3cabfd9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -49,6 +49,7 @@ jobs: with: context: . push: true + platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha diff --git a/.github/workflows/execution-env-publish.yml b/.github/workflows/execution-env-publish.yml index 2f1be9f..bafe021 100644 --- a/.github/workflows/execution-env-publish.yml +++ b/.github/workflows/execution-env-publish.yml @@ -92,6 +92,7 @@ jobs: context: docker file: docker/${{ matrix.language }}.Dockerfile push: true + platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha From ae1fb26b64c4126e87364d527ec8264ea6e5d4f5 Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 01:47:42 +0000 Subject: [PATCH 06/13] feat: Enhance GitHub workflows with manual triggers and improved image building logic - Added `workflow_dispatch` support to `docker-publish.yml` and `execution-env-publish.yml` for manual execution. - Introduced input option `build_all` in `execution-env-publish.yml` to force build all images. - Updated logic in `execution-env-publish.yml` to determine target languages based on workflow events and changes to the workflow file. --- .github/workflows/docker-publish.yml | 1 + .github/workflows/execution-env-publish.yml | 22 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 3cabfd9..89a4cb2 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -4,6 +4,7 @@ on: push: branches: ["main", "dev"] tags: ["v*.*.*"] + workflow_dispatch: env: REGISTRY: ghcr.io diff --git a/.github/workflows/execution-env-publish.yml b/.github/workflows/execution-env-publish.yml index bafe021..d58ebda 100644 --- a/.github/workflows/execution-env-publish.yml +++ b/.github/workflows/execution-env-publish.yml @@ -5,16 +5,24 @@ on: branches: ["main", "dev"] paths: - "docker/**" + - ".github/workflows/execution-env-publish.yml" + workflow_dispatch: + inputs: + build_all: + description: 'Force build all images' + type: boolean + default: false env: REGISTRY: ghcr.io IMAGE_BASE: ${{ github.repository }} + ALL_LANGUAGES: '["python", "nodejs", "go", "java", "c-cpp", "php", "rust", "fortran", "r", "d"]' jobs: filter: runs-on: ubuntu-latest outputs: - changes: ${{ steps.changes.outputs.changes }} + changes: ${{ steps.determine-targets.outputs.languages }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 @@ -44,6 +52,18 @@ jobs: d: - 'docker/d.Dockerfile' + - name: Determine targets + id: determine-targets + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ github.event.inputs.build_all }}" == "true" ]; then + echo "languages=${{ env.ALL_LANGUAGES }}" >> $GITHUB_OUTPUT + elif [ "${{ github.event_name }}" == "push" ] && echo "${{ github.event.head_commit.modified }}" | grep -q ".github/workflows/execution-env-publish.yml"; then + # If workflow itself changed, build everything + echo "languages=${{ env.ALL_LANGUAGES }}" >> $GITHUB_OUTPUT + else + echo "languages=${{ steps.changes.outputs.changes }}" >> $GITHUB_OUTPUT + fi + build-images: needs: filter if: ${{ needs.filter.outputs.changes != '[]' && needs.filter.outputs.changes != '' }} From 07a82c127f3cc82e2184cf391d8049e422997c8e Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 01:57:14 +0000 Subject: [PATCH 07/13] fix: Update execution-env-publish.yml for improved condition checks - Changed echo statements to use single quotes for consistency. - Updated conditional checks to use the `contains` function for better clarity when determining if the workflow file was modified. --- .github/workflows/execution-env-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/execution-env-publish.yml b/.github/workflows/execution-env-publish.yml index d58ebda..89188a4 100644 --- a/.github/workflows/execution-env-publish.yml +++ b/.github/workflows/execution-env-publish.yml @@ -56,10 +56,10 @@ jobs: id: determine-targets run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ github.event.inputs.build_all }}" == "true" ]; then - echo "languages=${{ env.ALL_LANGUAGES }}" >> $GITHUB_OUTPUT - elif [ "${{ github.event_name }}" == "push" ] && echo "${{ github.event.head_commit.modified }}" | grep -q ".github/workflows/execution-env-publish.yml"; then + echo 'languages=${{ env.ALL_LANGUAGES }}' >> $GITHUB_OUTPUT + elif [ "${{ github.event_name }}" == "push" ] && [ "${{ contains(github.event.head_commit.modified, '.github/workflows/execution-env-publish.yml') }}" == "true" ]; then # If workflow itself changed, build everything - echo "languages=${{ env.ALL_LANGUAGES }}" >> $GITHUB_OUTPUT + echo 'languages=${{ env.ALL_LANGUAGES }}' >> $GITHUB_OUTPUT else echo "languages=${{ steps.changes.outputs.changes }}" >> $GITHUB_OUTPUT fi From dfcfdee5198928d8cf9de599799b7caad523f24f Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 01:59:53 +0000 Subject: [PATCH 08/13] feat: Update execution-env-publish.yml to enhance rebuild logic - Added a new filter for `rebuild_all` to trigger builds when specific files change. - Improved conditional checks to determine when to build all images based on workflow changes. - Filtered out `rebuild_all` from the changes list for cleaner output. --- .github/workflows/execution-env-publish.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/execution-env-publish.yml b/.github/workflows/execution-env-publish.yml index 89188a4..8b681e0 100644 --- a/.github/workflows/execution-env-publish.yml +++ b/.github/workflows/execution-env-publish.yml @@ -29,6 +29,9 @@ jobs: id: changes with: filters: | + rebuild_all: + - '.github/workflows/execution-env-publish.yml' + - 'docker/entrypoint.sh' python: - 'docker/python.Dockerfile' - 'docker/repl_server.py' @@ -57,11 +60,14 @@ jobs: run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ "${{ github.event.inputs.build_all }}" == "true" ]; then echo 'languages=${{ env.ALL_LANGUAGES }}' >> $GITHUB_OUTPUT - elif [ "${{ github.event_name }}" == "push" ] && [ "${{ contains(github.event.head_commit.modified, '.github/workflows/execution-env-publish.yml') }}" == "true" ]; then - # If workflow itself changed, build everything + elif [ "${{ steps.changes.outputs.rebuild_all }}" == "true" ]; then + # If workflow or shared files changed, build everything echo 'languages=${{ env.ALL_LANGUAGES }}' >> $GITHUB_OUTPUT else - echo "languages=${{ steps.changes.outputs.changes }}" >> $GITHUB_OUTPUT + # Filter out rebuild_all from the changes list + CHANGES='${{ steps.changes.outputs.changes }}' + FILTERED=$(echo "$CHANGES" | jq -c '[.[] | select(. != "rebuild_all")]') + echo "languages=$FILTERED" >> $GITHUB_OUTPUT fi build-images: From cd1a7d0ad616a018d309a9827f26d863401c3c11 Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 03:13:30 +0000 Subject: [PATCH 09/13] feat: Update Docker image configuration for execution environment - Changed the Docker image tag in `docker-compose.ghcr.yml` from `latest` to `dev` for development purposes. - Enhanced the `Settings` class to include a new field for `docker_image_tag`, allowing dynamic specification of image tags. - Modified the `get_image_for_language` function in `languages.py` to accept a tag parameter, improving flexibility in image retrieval. --- docker-compose.ghcr.yml | 3 ++- src/config/__init__.py | 6 +++++- src/config/languages.py | 8 ++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docker-compose.ghcr.yml b/docker-compose.ghcr.yml index 7e9a7d8..8e58137 100644 --- a/docker-compose.ghcr.yml +++ b/docker-compose.ghcr.yml @@ -1,7 +1,7 @@ services: # Code Interpreter API api: - image: ghcr.io/usnavy13/librecodeinterpreter:latest + image: ghcr.io/usnavy13/librecodeinterpreter:dev container_name: code-interpreter-api user: "1000:988" # Run as user with docker group access cap_add: @@ -16,6 +16,7 @@ services: - API_HOST=0.0.0.0 - API_PORT=8000 - DOCKER_IMAGE_REGISTRY=ghcr.io/usnavy13/librecodeinterpreter + - DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-latest} # Service discovery (container names) - REDIS_HOST=redis diff --git a/src/config/__init__.py b/src/config/__init__.py index d0b15e4..0c6b8b5 100644 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -115,6 +115,10 @@ class Settings(BaseSettings): default="code-interpreter", description="Registry/namespace prefix for execution environment images", ) + docker_image_tag: str = Field( + default="latest", + description="Tag for execution environment images (e.g. 'latest', 'dev')", + ) docker_timeout: int = Field(default=60, ge=10) docker_network_mode: str = Field(default="none") docker_security_opt: List[str] = Field( @@ -599,7 +603,7 @@ def get_image_for_language(self, code: str) -> str: # Fallback to languages.py logic if not in settings from .languages import get_image_for_language as get_img - return get_img(code, registry=self.docker_image_registry) + return get_img(code, registry=self.docker_image_registry, tag=self.docker_image_tag) def get_execution_timeout(self, language: str) -> int: """Get execution timeout for a specific language.""" diff --git a/src/config/languages.py b/src/config/languages.py index a00a4d4..3e0602f 100644 --- a/src/config/languages.py +++ b/src/config/languages.py @@ -183,11 +183,15 @@ def is_supported_language(code: str) -> bool: # Convenience lookups for backward compatibility during transition -def get_image_for_language(code: str, registry: Optional[str] = None) -> str: +def get_image_for_language(code: str, registry: Optional[str] = None, tag: str = "latest") -> str: """Get Docker image for a language.""" lang = get_language(code) if lang: - return f"{registry}/{lang.image}" if registry else lang.image + # Extract base image name without the default :latest tag + base_image = lang.image.rsplit(":", 1)[0] + if registry: + return f"{registry}/{base_image}:{tag}" + return f"{base_image}:{tag}" raise ValueError(f"Unsupported language: {code}") From 4822649e0f5915c935a9d4529a08afa476fa1934 Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 03:33:33 +0000 Subject: [PATCH 10/13] feat: Update Docker image tag handling for development environment - Changed the default Docker image tag in `docker-compose.ghcr.yml` from `latest` to `dev` to better suit development needs. - Enhanced the `Settings` class to retrieve the Docker image tag dynamically, allowing for more flexible image management. --- docker-compose.ghcr.yml | 2 +- src/config/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-compose.ghcr.yml b/docker-compose.ghcr.yml index 8e58137..760aaee 100644 --- a/docker-compose.ghcr.yml +++ b/docker-compose.ghcr.yml @@ -16,7 +16,7 @@ services: - API_HOST=0.0.0.0 - API_PORT=8000 - DOCKER_IMAGE_REGISTRY=ghcr.io/usnavy13/librecodeinterpreter - - DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-latest} + - DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-dev} # Service discovery (container names) - REDIS_HOST=redis diff --git a/src/config/__init__.py b/src/config/__init__.py index 0c6b8b5..03448ea 100644 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -401,9 +401,10 @@ def _set_supported_languages(cls, v, values): return v registry = values.get("docker_image_registry", "code-interpreter") + tag = values.get("docker_image_tag", "latest") return { code: { - "image": f"{registry}/{lang.image}" if registry else lang.image, + "image": f"{registry}/{lang.image.rsplit(':', 1)[0]}:{tag}" if registry else f"{lang.image.rsplit(':', 1)[0]}:{tag}", "timeout_multiplier": lang.timeout_multiplier, "memory_multiplier": lang.memory_multiplier, } From 8f182359b033a43f4d0d8a686cf9928a2a63c06b Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 04:00:05 +0000 Subject: [PATCH 11/13] feat: Update SSL configuration in environment files and documentation - Modified `.env.example` to clarify SSL certificate paths for Docker and non-Docker deployments. - Updated `docker-compose.ghcr.yml` and `docker-compose.yml` to use a dynamic path for SSL certificates, enhancing flexibility. - Revised `CONFIGURATION.md` to provide detailed instructions for SSL setup in both Docker and non-Docker environments. --- .env.example | 13 ++++++++--- docker-compose.ghcr.yml | 6 ++--- docker-compose.yml | 2 +- docs/CONFIGURATION.md | 50 +++++++++++++++++++++++++++++++---------- 4 files changed, 52 insertions(+), 19 deletions(-) diff --git a/.env.example b/.env.example index f87fe23..bd92b13 100644 --- a/.env.example +++ b/.env.example @@ -9,10 +9,17 @@ API_RELOAD=false # SSL/HTTPS Configuration ENABLE_HTTPS=false HTTPS_PORT=443 -SSL_CERT_FILE=/app/ssl/cert.pem -SSL_KEY_FILE=/app/ssl/key.pem SSL_REDIRECT=false -# SSL_CA_CERTS=/app/ssl/ca.pem # Optional CA certificates + +# Docker: Path to directory containing cert.pem and key.pem on the host +# The directory is mounted to /app/ssl/ inside the container automatically. +# Default is ./ssl (relative to docker-compose.yml) +# SSL_CERTS_PATH=/path/to/your/ssl/certs + +# Non-Docker only: Absolute paths to certificate files (not needed for Docker) +# SSL_CERT_FILE=/path/to/cert.pem +# SSL_KEY_FILE=/path/to/key.pem +# SSL_CA_CERTS=/path/to/ca.pem # Authentication Configuration API_KEY=your-secure-api-key-here-change-this-in-production diff --git a/docker-compose.ghcr.yml b/docker-compose.ghcr.yml index 760aaee..1522743 100644 --- a/docker-compose.ghcr.yml +++ b/docker-compose.ghcr.yml @@ -1,7 +1,7 @@ services: # Code Interpreter API api: - image: ghcr.io/usnavy13/librecodeinterpreter:dev + image: ghcr.io/usnavy13/librecodeinterpreter:latest container_name: code-interpreter-api user: "1000:988" # Run as user with docker group access cap_add: @@ -16,7 +16,7 @@ services: - API_HOST=0.0.0.0 - API_PORT=8000 - DOCKER_IMAGE_REGISTRY=ghcr.io/usnavy13/librecodeinterpreter - - DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-dev} + - DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-latest} # Service discovery (container names) - REDIS_HOST=redis @@ -33,7 +33,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock - ./logs:/app/logs - ./data:/app/data - - ./ssl:/app/ssl + - ${SSL_CERTS_PATH:-./ssl}:/app/ssl depends_on: - redis - minio diff --git a/docker-compose.yml b/docker-compose.yml index 5e7b691..1f3703d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock - ./logs:/app/logs - ./data:/app/data - - ./ssl:/app/ssl + - ${SSL_CERTS_PATH:-./ssl}:/app/ssl - ./dashboard:/app/dashboard - ./src:/app/src depends_on: diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index cd696fd..f277221 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -43,16 +43,29 @@ Controls the basic API server settings. Configures SSL/TLS support for secure HTTPS connections. -| Variable | Default | Description | -| --------------- | ------- | ------------------------------------------- | -| `ENABLE_HTTPS` | `false` | Enable HTTPS/SSL support | -| `HTTPS_PORT` | `443` | HTTPS server port | -| `SSL_CERT_FILE` | - | Path to SSL certificate file (.crt or .pem) | -| `SSL_KEY_FILE` | - | Path to SSL private key file (.key) | -| `SSL_REDIRECT` | `false` | Redirect HTTP traffic to HTTPS | -| `SSL_CA_CERTS` | - | Path to CA certificates file (optional) | +#### Docker Deployments -**HTTPS Setup:** +| Variable | Default | Description | +| ---------------- | -------- | -------------------------------------------------------- | +| `ENABLE_HTTPS` | `false` | Enable HTTPS/SSL support | +| `HTTPS_PORT` | `443` | HTTPS server port | +| `SSL_CERTS_PATH` | `./ssl` | Host path to directory containing `cert.pem` and `key.pem` | +| `SSL_REDIRECT` | `false` | Redirect HTTP traffic to HTTPS | + +> **Note:** When using Docker, the certificate files are automatically mapped to `/app/ssl/` inside the container. You only need to set `SSL_CERTS_PATH` to point to your certificates directory on the host. + +#### Non-Docker Deployments + +| Variable | Default | Description | +| ---------------- | -------- | -------------------------------------------------------- | +| `ENABLE_HTTPS` | `false` | Enable HTTPS/SSL support | +| `HTTPS_PORT` | `443` | HTTPS server port | +| `SSL_CERT_FILE` | - | Absolute path to SSL certificate file (.pem) | +| `SSL_KEY_FILE` | - | Absolute path to SSL private key file (.pem) | +| `SSL_CA_CERTS` | - | Path to CA certificates file (optional) | +| `SSL_REDIRECT` | `false` | Redirect HTTP traffic to HTTPS | + +**HTTPS Setup (Docker):** 1. **Generate or obtain SSL certificates**: @@ -69,17 +82,30 @@ Configures SSL/TLS support for secure HTTPS connections. ```bash ENABLE_HTTPS=true HTTPS_PORT=443 - SSL_CERT_FILE=/app/ssl/cert.pem - SSL_KEY_FILE=/app/ssl/key.pem SSL_REDIRECT=true # Optional: redirect HTTP to HTTPS + + # If using the default ./ssl directory, no additional config needed. + # If your certs are elsewhere, set the path: + # SSL_CERTS_PATH=/path/to/your/ssl/certs ``` + The directory must contain files named `cert.pem` and `key.pem`. + 3. **Deploy with Docker Compose**: ```bash - # Make sure SSL certificates are in ./ssl/ directory docker-compose up -d ``` +**HTTPS Setup (Non-Docker):** + +```bash +ENABLE_HTTPS=true +HTTPS_PORT=443 +SSL_CERT_FILE=/absolute/path/to/cert.pem +SSL_KEY_FILE=/absolute/path/to/key.pem +SSL_REDIRECT=true +``` + **Security Notes:** - Use certificates from trusted Certificate Authorities in production From 090f564e44bf1ec3ba1f03422395f2cd46d1f706 Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 04:06:22 +0000 Subject: [PATCH 12/13] refactor: Improve code readability and formatting in configuration and service files - Enhanced the `Settings` class in `__init__.py` for better clarity in Docker image retrieval logic. - Reformatted the `get_image_for_language` function in `languages.py` for improved readability. - Adjusted the `ApiKeyRecord` class in `api_key.py` to streamline dictionary comprehension. - Improved formatting in the `HealthCheckService` class in `health.py` for better alignment. - Refactored the `ExecutionContext` class in `orchestrator.py` to enhance code structure. --- src/config/__init__.py | 13 +++++++++---- src/config/languages.py | 4 +++- src/models/api_key.py | 6 +++--- src/services/health.py | 4 +++- src/services/orchestrator.py | 6 +++--- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/config/__init__.py b/src/config/__init__.py index 03448ea..d02d98d 100644 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -399,12 +399,14 @@ def _set_supported_languages(cls, v, values): """Initialize supported_languages with registry-prefixed images.""" if v: return v - + registry = values.get("docker_image_registry", "code-interpreter") tag = values.get("docker_image_tag", "latest") return { code: { - "image": f"{registry}/{lang.image.rsplit(':', 1)[0]}:{tag}" if registry else f"{lang.image.rsplit(':', 1)[0]}:{tag}", + "image": f"{registry}/{lang.image.rsplit(':', 1)[0]}:{tag}" + if registry + else f"{lang.image.rsplit(':', 1)[0]}:{tag}", "timeout_multiplier": lang.timeout_multiplier, "memory_multiplier": lang.memory_multiplier, } @@ -601,10 +603,13 @@ def get_image_for_language(self, code: str) -> str: config = self.get_language_config(code) if config and "image" in config: return config["image"] - + # Fallback to languages.py logic if not in settings from .languages import get_image_for_language as get_img - return get_img(code, registry=self.docker_image_registry, tag=self.docker_image_tag) + + return get_img( + code, registry=self.docker_image_registry, tag=self.docker_image_tag + ) def get_execution_timeout(self, language: str) -> int: """Get execution timeout for a specific language.""" diff --git a/src/config/languages.py b/src/config/languages.py index 3e0602f..68bc671 100644 --- a/src/config/languages.py +++ b/src/config/languages.py @@ -183,7 +183,9 @@ def is_supported_language(code: str) -> bool: # Convenience lookups for backward compatibility during transition -def get_image_for_language(code: str, registry: Optional[str] = None, tag: str = "latest") -> str: +def get_image_for_language( + code: str, registry: Optional[str] = None, tag: str = "latest" +) -> str: """Get Docker image for a language.""" lang = get_language(code) if lang: diff --git a/src/models/api_key.py b/src/models/api_key.py index e4d7320..51c971e 100644 --- a/src/models/api_key.py +++ b/src/models/api_key.py @@ -111,9 +111,9 @@ def from_redis_hash(cls, data: Dict[bytes, bytes]) -> "ApiKeyRecord": """Create from Redis hash data (bytes keys/values).""" # Decode bytes to strings decoded = { - k.decode() if isinstance(k, bytes) else k: ( - v.decode() if isinstance(v, bytes) else v - ) + k.decode() + if isinstance(k, bytes) + else k: (v.decode() if isinstance(v, bytes) else v) for k, v in data.items() } diff --git a/src/services/health.py b/src/services/health.py index b09ec3c..fdd50d0 100644 --- a/src/services/health.py +++ b/src/services/health.py @@ -404,7 +404,9 @@ async def check_docker(self) -> HealthCheckResult: "running_containers": running_containers, "registry_accessible": registry_accessible, "server_version": system_info.get("ServerVersion", "unknown"), - "memory_total_gb": round(system_info.get("MemTotal", 0) / (1024**3), 2), + "memory_total_gb": round( + system_info.get("MemTotal", 0) / (1024**3), 2 + ), "cpu_count": system_info.get("NCPU", 0), } diff --git a/src/services/orchestrator.py b/src/services/orchestrator.py index f58614d..b78a24a 100644 --- a/src/services/orchestrator.py +++ b/src/services/orchestrator.py @@ -62,9 +62,9 @@ class ExecutionContext: generated_files: Optional[List[FileRef]] = None stdout: str = "" stderr: str = "" - container: Optional[Any] = ( - None # Container used for execution (avoids session lookup) - ) + container: Optional[ + Any + ] = None # Container used for execution (avoids session lookup) # State persistence fields initial_state: Optional[str] = None new_state: Optional[str] = None From a14ddeaa1a4ff128ffe0c96aa91b34871536a944 Mon Sep 17 00:00:00 2001 From: Joe Licata Date: Sat, 27 Dec 2025 04:08:55 +0000 Subject: [PATCH 13/13] refactor: Enhance code readability and formatting across multiple files - Improved the formatting of the Docker image retrieval logic in the `Settings` class in `__init__.py`. - Streamlined dictionary comprehension in the `ApiKeyRecord` class in `api_key.py`. - Refined the formatting of memory total calculation in the `HealthCheckService` class in `health.py`. - Enhanced the structure of the `ExecutionContext` class in `orchestrator.py` for better clarity. --- src/config/__init__.py | 8 +++++--- src/models/api_key.py | 6 +++--- src/services/health.py | 4 +--- src/services/orchestrator.py | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/config/__init__.py b/src/config/__init__.py index d02d98d..510d2c5 100644 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -404,9 +404,11 @@ def _set_supported_languages(cls, v, values): tag = values.get("docker_image_tag", "latest") return { code: { - "image": f"{registry}/{lang.image.rsplit(':', 1)[0]}:{tag}" - if registry - else f"{lang.image.rsplit(':', 1)[0]}:{tag}", + "image": ( + f"{registry}/{lang.image.rsplit(':', 1)[0]}:{tag}" + if registry + else f"{lang.image.rsplit(':', 1)[0]}:{tag}" + ), "timeout_multiplier": lang.timeout_multiplier, "memory_multiplier": lang.memory_multiplier, } diff --git a/src/models/api_key.py b/src/models/api_key.py index 51c971e..e4d7320 100644 --- a/src/models/api_key.py +++ b/src/models/api_key.py @@ -111,9 +111,9 @@ def from_redis_hash(cls, data: Dict[bytes, bytes]) -> "ApiKeyRecord": """Create from Redis hash data (bytes keys/values).""" # Decode bytes to strings decoded = { - k.decode() - if isinstance(k, bytes) - else k: (v.decode() if isinstance(v, bytes) else v) + k.decode() if isinstance(k, bytes) else k: ( + v.decode() if isinstance(v, bytes) else v + ) for k, v in data.items() } diff --git a/src/services/health.py b/src/services/health.py index fdd50d0..b09ec3c 100644 --- a/src/services/health.py +++ b/src/services/health.py @@ -404,9 +404,7 @@ async def check_docker(self) -> HealthCheckResult: "running_containers": running_containers, "registry_accessible": registry_accessible, "server_version": system_info.get("ServerVersion", "unknown"), - "memory_total_gb": round( - system_info.get("MemTotal", 0) / (1024**3), 2 - ), + "memory_total_gb": round(system_info.get("MemTotal", 0) / (1024**3), 2), "cpu_count": system_info.get("NCPU", 0), } diff --git a/src/services/orchestrator.py b/src/services/orchestrator.py index b78a24a..f58614d 100644 --- a/src/services/orchestrator.py +++ b/src/services/orchestrator.py @@ -62,9 +62,9 @@ class ExecutionContext: generated_files: Optional[List[FileRef]] = None stdout: str = "" stderr: str = "" - container: Optional[ - Any - ] = None # Container used for execution (avoids session lookup) + container: Optional[Any] = ( + None # Container used for execution (avoids session lookup) + ) # State persistence fields initial_state: Optional[str] = None new_state: Optional[str] = None