Harden backend persistence and packaging

Makefile

@@ -6,11 +6,7 @@ DESKTOPDIR = $(PREFIX)/share/applications
 METAINFODIR = $(PREFIX)/share/metainfo
 LICENSEDIR = $(PREFIX)/share/licenses/openbox
 
-PYTHON_SOURCES = openbox.py web_app.py openbox_logging.py importers.py arcade.py catalog.py cloud_sync.py \
-	emulators.py retroachievements.py plugins.py plugin_runner.py metadata.py \
-	archives.py saves.py updates.py env_config.py parity_discovery.py parity_import.py \
-	parity_integrations.py parity_media.py parity_saves.py parity_storefront.py plugin_catalog.py parity_premium.py stock_themes.py parity_gameyfin.py parity_save_tools.py \
-	parity_filter_presets.py parity_deeplinks.py parity_backup.py parity_tracking.py parity_igdb.py parity_emulator_defs.py parity_import_policy.py parity_gamescope.py
+PYTHON_SOURCES = $(shell sed '/^[[:space:]]*#/d;/^[[:space:]]*$$/d' runtime_modules.txt)
 
 DATA_FILES = index.html openbox.svg openbox.metainfo.xml LICENSE
 

archives.py

@@ -20,9 +20,29 @@ def safe_zip_extract(archive, destination):
             target = (destination / info.filename).resolve()
             if root != target and root not in target.parents:
                 raise ValueError(f"Unsafe archive path: {info.filename}")
+            mode = (info.external_attr >> 16) & 0o170000
+            if mode == 0o120000:
+                raise ValueError(f"Archive symlinks are not supported: {info.filename}")
         package.extractall(destination)
 
 
+def validate_7z_paths(extractor, archive):
+    result = subprocess.run(
+        [extractor, "l", "-slt", str(archive)],
+        check=True,
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+    for line in result.stdout.splitlines():
+        if not line.startswith("Path = "):
+            continue
+        name = line[7:]
+        candidate = Path(name)
+        if candidate.is_absolute() or ".." in candidate.parts:
+            raise ValueError(f"Unsafe archive path: {name}")
+
+
 def choose_game_file(destination, member=""):
     if member:
         selected = (destination / member).resolve()
@@ -48,6 +68,7 @@ def extract_game(archive_path, cache_root, member=""):
             extractor = shutil.which("7z") or shutil.which("7zz")
             if not extractor:
                 raise FileNotFoundError("7z or 7zz is required to extract this archive.")
+            validate_7z_paths(extractor, archive)
             subprocess.run([extractor, "x", "-y", f"-o{destination}", str(archive)], check=True, capture_output=True)
         complete.touch()
     return choose_game_file(destination, member)

backend_io.py

@@ -0,0 +1,97 @@
+"""Shared bounded network and filesystem helpers used by backend operations."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from urllib.request import Request, urlopen
+
+
+DEFAULT_MAX_DOWNLOAD = 64 * 1024 * 1024
+CHUNK_SIZE = 1024 * 1024
+
+
+def contained_path(path: Path, roots, *, must_exist=False) -> Path:
+    candidate = Path(path).expanduser().resolve(strict=False)
+    allowed = [Path(root).expanduser().resolve(strict=False) for root in roots]
+    if not any(candidate == root or root in candidate.parents for root in allowed):
+        raise ValueError(f"Path is outside an approved OpenBox directory: {candidate}")
+    if must_exist and not candidate.exists():
+        raise FileNotFoundError(str(candidate))
+    return candidate
+
+
+def safe_media_path(path: Path, data_root: Path) -> Path:
+    return contained_path(path, [Path(data_root)])
+
+
+def remove_file_if_safe(path: Path, data_root: Path) -> bool:
+    target = safe_media_path(path, data_root)
+    if not target.is_file():
+        return False
+    target.unlink()
+    return True
+
+
+def download_file(
+    url: str,
+    destination: Path,
+    *,
+    expected_types=(),
+    max_bytes=DEFAULT_MAX_DOWNLOAD,
+    timeout=30,
+    opener=urlopen,
+    headers=None,
+    sha256="",
+) -> Path:
+    request = Request(url, headers={"User-Agent": "OpenBox/1", **(headers or {})})
+    destination = Path(destination)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    temporary_name = None
+    digest = hashlib.sha256()
+    try:
+        with opener(request, timeout=timeout) as response:
+            if response.headers and hasattr(response.headers, "get_content_type"):
+                content_type = response.headers.get_content_type()
+            else:
+                content_type = str(response.headers.get("Content-Type", "")).split(";", 1)[0] if response.headers else ""
+            if expected_types and not any(content_type.startswith(item) for item in expected_types):
+                raise ValueError(f"The remote server returned an unsupported content type: {content_type or 'unknown'}")
+            try:
+                declared = int(response.headers.get("Content-Length", "0"))
+            except (TypeError, ValueError):
+                declared = 0
+            if declared > max_bytes:
+                raise ValueError("The download is too large.")
+            fd, temporary_name = tempfile.mkstemp(prefix=f".{destination.name}.", suffix=".tmp", dir=destination.parent)
+            with os.fdopen(fd, "wb") as output:
+                total = 0
+                while True:
+                    chunk = response.read(CHUNK_SIZE)
+                    if not chunk:
+                        break
+                    total += len(chunk)
+                    if total > max_bytes:
+                        raise ValueError("The download is too large.")
+                    digest.update(chunk)
+                    output.write(chunk)
+                output.flush()
+                os.fsync(output.fileno())
+        if sha256 and digest.hexdigest().casefold() != str(sha256).casefold():
+            raise ValueError("The downloaded file failed checksum verification.")
+        os.replace(temporary_name, destination)
+        temporary_name = None
+    finally:
+        if temporary_name:
+            Path(temporary_name).unlink(missing_ok=True)
+    return destination
+
+
+def safe_copytree(source: Path, destination: Path) -> None:
+    source = Path(source)
+    destination = Path(destination)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copytree(source, destination)

build_appimage.sh

@@ -17,9 +17,11 @@ cp "$python_binary" "$appdir/usr/bin/python3"
 cp -a "$stdlib" "$appdir/usr/lib/python$python_version"
 find "$appdir/usr/lib/python$python_version" -type d -name __pycache__ -prune -exec rm -rf -- {} +
 
-for file in openbox.py web_app.py importers.py arcade.py catalog.py cloud_sync.py emulators.py retroachievements.py plugins.py plugin_runner.py metadata.py archives.py saves.py updates.py env_config.py parity_discovery.py parity_import.py parity_integrations.py parity_media.py parity_saves.py parity_storefront.py plugin_catalog.py parity_premium.py stock_themes.py parity_gameyfin.py parity_save_tools.py parity_filter_presets.py parity_deeplinks.py parity_backup.py parity_tracking.py parity_igdb.py parity_emulator_defs.py parity_import_policy.py parity_gamescope.py index.html; do
+while IFS= read -r file; do
+  [ -n "$file" ] || continue
   cp "$source_root/$file" "$appdir/usr/share/openbox/$file"
-done
+done < "$source_root/runtime_modules.txt"
+cp "$source_root/index.html" "$appdir/usr/share/openbox/index.html"
 mkdir -p "$appdir/usr/share/openbox/emulator_defs"
 cp "$source_root"/emulator_defs/*.yaml "$appdir/usr/share/openbox/emulator_defs/"
 install -Dm755 "$source_root/scripts/openbox-launcher.sh" "$appdir/usr/share/openbox/openbox-launcher.sh"

catalog.py

@@ -56,7 +56,11 @@ def bulk_update(games, ids, changes):
             clean[field] = {str(key).strip(): str(val).strip() for key, val in value.items() if str(key).strip()}
         else:
             clean[field] = str(value).strip()
-    selected = sorted(set(int(index) for index in ids))
+    stable_indexes = {str(game.get("game_id")): index for index, game in enumerate(games) if game.get("game_id")}
+    selected = sorted({
+        stable_indexes[str(value)] if str(value) in stable_indexes else int(value)
+        for value in ids
+    })
     if selected[0] < 0 or selected[-1] >= len(games):
         raise IndexError("A selected game no longer exists.")
     for index in selected:

cloud_sync.py

@@ -16,6 +16,12 @@ def nonnegative_int(value):
 
 
 def game_key(game):
+    if game.get("game_id"):
+        return f"id:{game['game_id']}"
+    return legacy_game_key(game)
+
+
+def legacy_game_key(game):
     if game.get("steam_app_id"):
         return f"steam:{game['steam_app_id']}"
     if game.get("heroic_app_id"):
@@ -42,6 +48,8 @@ def sync_statistics(state, folder, now=None):
     changed = 0
     for game in state["games"]:
         saved = remote_games.get(game_key(game), {})
+        if not saved:
+            saved = remote_games.get(legacy_game_key(game), {})
         if not isinstance(saved, dict):
             continue
         before = {field:game.get(field) for field in STAT_FIELDS}

io.openbox.GameLauncher.yml

@@ -20,7 +20,8 @@ modules:
     buildsystem: simple
     build-commands:
       - mkdir -p /app/share/openbox
-      - for f in openbox.py web_app.py openbox_logging.py importers.py arcade.py catalog.py cloud_sync.py emulators.py retroachievements.py plugins.py plugin_runner.py metadata.py archives.py saves.py updates.py env_config.py parity_discovery.py parity_import.py parity_integrations.py parity_media.py parity_saves.py parity_storefront.py plugin_catalog.py parity_premium.py stock_themes.py parity_gameyfin.py parity_save_tools.py parity_filter_presets.py parity_deeplinks.py parity_backup.py parity_tracking.py parity_igdb.py parity_emulator_defs.py parity_import_policy.py parity_gamescope.py index.html; do install -Dm644 "$f" "/app/share/openbox/$f"; done
+      - while IFS= read -r f; do [ -n "$f" ] || continue; install -Dm644 "$f" "/app/share/openbox/$f"; done < runtime_modules.txt
+      - install -Dm644 index.html /app/share/openbox/index.html
       - install -d /app/share/openbox/emulator_defs
       - for f in emulator_defs/*.yaml; do install -Dm644 "$f" "/app/share/openbox/emulator_defs/"; done
       - install -Dm755 scripts/openbox-launcher.sh /app/share/openbox/openbox-launcher.sh

job_manager.py

@@ -0,0 +1,70 @@
+"""Small standard-library job manager for bounded backend work."""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from datetime import datetime, timezone
+from typing import Callable
+
+
+LOGGER = logging.getLogger("openbox.jobs")
+
+
+class JobManager:
+    def __init__(self):
+        self._lock = threading.RLock()
+        self._jobs = {}
+
+    def snapshot(self, name):
+        with self._lock:
+            return dict(self._jobs.get(name, {}))
+
+    def submit(self, name, worker: Callable[[], dict], *, replace=False):
+        with self._lock:
+            current = self._jobs.get(name, {})
+            if current.get("state") in {"queued", "running"} and not replace:
+                return dict(current)
+            now = datetime.now(timezone.utc).isoformat()
+            job = {
+                "name": name,
+                "state": "queued",
+                "started_at": "",
+                "finished_at": "",
+                "created_at": now,
+                "error": "",
+                "attempt": int(current.get("attempt", 0)) + 1,
+            }
+            self._jobs[name] = job
+
+        def run():
+            with self._lock:
+                self._jobs[name].update({
+                    "state": "running",
+                    "started_at": datetime.now(timezone.utc).isoformat(),
+                })
+            started = time.monotonic()
+            try:
+                result = worker() or {}
+                if not isinstance(result, dict):
+                    result = {"result": result}
+                with self._lock:
+                    self._jobs[name].update(result)
+                    self._jobs[name].update({
+                        "state": "done",
+                        "finished_at": datetime.now(timezone.utc).isoformat(),
+                        "duration_seconds": round(time.monotonic() - started, 3),
+                    })
+            except Exception as error:
+                LOGGER.exception("Backend job %s failed", name)
+                with self._lock:
+                    self._jobs[name].update({
+                        "state": "error",
+                        "error": str(error),
+                        "finished_at": datetime.now(timezone.utc).isoformat(),
+                        "duration_seconds": round(time.monotonic() - started, 3),
+                    })
+
+        threading.Thread(target=run, name=f"openbox-job-{name}", daemon=True).start()
+        return dict(job)

metadata.py

@@ -9,6 +9,8 @@
 from urllib.request import Request, urlopen
 from xml.etree import ElementTree
 
+from backend_io import download_file
+
 
 DATABASE_URL = "https://gamesdb.launchbox-app.com/Metadata.zip"
 IMAGE_URL = "https://images.launchbox-app.com/"
@@ -82,11 +84,15 @@ def build_database(metadata_zip, destination):
 def sync_database(destination, opener=urlopen):
     destination = Path(destination)
     destination.parent.mkdir(parents=True, exist_ok=True)
-    request = Request(DATABASE_URL, headers={"User-Agent":"OpenBox/1"})
     with tempfile.NamedTemporaryFile(dir=destination.parent, suffix=".zip", delete=False) as temporary:
         archive = Path(temporary.name)
-        with opener(request, timeout=120) as response:
-            shutil.copyfileobj(response, temporary)
+    download_file(
+        DATABASE_URL,
+        archive,
+        max_bytes=2 * 1024 * 1024 * 1024,
+        timeout=120,
+        opener=opener,
+    )
     try:
         build_database(archive, destination)
     finally:
@@ -110,16 +116,14 @@ def search_games(database_path, title, platform="", limit=20):
 
 def download_image(filename, destination, opener=urlopen):
     filename = Path(filename).name
-    request = Request(IMAGE_URL + filename, headers={"User-Agent":"OpenBox/1"})
-    with opener(request, timeout=30) as response:
-        if not response.headers.get_content_type().startswith("image/"):
-            raise ValueError("The metadata server did not return an image.")
-        destination.parent.mkdir(parents=True, exist_ok=True)
-        temporary = destination.with_suffix(destination.suffix + ".tmp")
-        with temporary.open("wb") as output:
-            shutil.copyfileobj(response, output)
-        temporary.replace(destination)
-    return str(destination)
+    return str(download_file(
+        IMAGE_URL + filename,
+        destination,
+        expected_types=("image/",),
+        max_bytes=32 * 1024 * 1024,
+        timeout=30,
+        opener=opener,
+    ))
 
 
 def apply_game_metadata(game, database_path, database_id, media_types, media_root, overwrite=False, opener=urlopen, region_priority=None):

openbox.py

@@ -17,6 +17,7 @@
 
 from archives import extract_game
 from openbox_logging import configure_logging
+from state_store import JsonStateStore
 
 CUSTOM_DATA_DIR = os.environ.get("OPENBOX_DATA_DIR")
 APP_DIR = Path(CUSTOM_DATA_DIR or Path.home() / ".local/share/openbox-game-launcher").expanduser()
@@ -27,6 +28,8 @@
     shutil.copy2(LEGACY_DATA, DATA)
 from parity_import import EXTENSIONS_EXTRA, PLATFORM_BY_EXTENSION_EXTRA
 
+STATE_STORE = JsonStateStore(DATA)
+
 EXTENSIONS = {".sh", ".appimage", ".exe", ".iso", ".rom", ".nes", ".sfc", ".smc", ".gba", ".gb", ".gbc", ".zip", ".7z", ".rar"} | EXTENSIONS_EXTRA
 PLATFORM_BY_EXTENSION = {
     ".nes": "NES", ".sfc": "SNES", ".smc": "SNES", ".gba": "Game Boy Advance",
@@ -59,27 +62,20 @@ def purge_demo_games(state):
 
 
 def load_state():
-    try:
-        raw = json.loads(DATA.read_text())
-        if isinstance(raw, list):
-            return {"games": raw, "profiles": {}, "history": []}
-        if not isinstance(raw, dict):
-            raise AttributeError
-        raw.setdefault("games", [])
-        raw.setdefault("profiles", {})
-        raw.setdefault("history", [])
-        raw.setdefault("settings", {})
-        raw.setdefault("playlists", [])
-        return raw
-    except (FileNotFoundError, json.JSONDecodeError, AttributeError):
-        return {"games": [], "profiles": {}, "history": [], "settings": {}, "playlists": []}
+    return STATE_STORE.load()
 
 
 def save_state(state):
-    DATA.parent.mkdir(parents=True, exist_ok=True)
-    temporary = DATA.with_suffix(".tmp")
-    temporary.write_text(json.dumps(state, indent=2))
-    temporary.replace(DATA)
+    return STATE_STORE.save(state)
+
+
+def update_state(mutator):
+    """Apply one state mutation under the cross-process transaction lock."""
+    return STATE_STORE.update(mutator)
+
+
+def recover_state():
+    return STATE_STORE.recover()
 
 
 def format_duration(seconds):