From 9d5363a3c8f7c5e333e38824c495f38a3ec4ea30 Mon Sep 17 00:00:00 2001 From: vindeckyy Date: Thu, 30 Jul 2026 20:54:23 -0400 Subject: [PATCH] Harden backend persistence and packaging --- Makefile | 6 +- archives.py | 21 ++ backend_io.py | 97 +++++++ build_appimage.sh | 6 +- catalog.py | 6 +- cloud_sync.py | 8 + io.openbox.GameLauncher.yml | 3 +- job_manager.py | 70 +++++ metadata.py | 30 ++- openbox.py | 32 +-- parity_gameyfin.py | 18 +- parity_integrations.py | 37 +-- parity_media.py | 8 +- parity_premium.py | 14 +- parity_saves.py | 26 +- plugin_catalog.py | 13 +- plugins.py | 19 +- retroachievements.py | 8 +- runtime_modules.txt | 38 +++ saves.py | 5 +- state_store.py | 210 +++++++++++++++ test_backend_hardening.py | 101 ++++++++ test_bug_sweep_api.py | 1 + test_packaging.py | 38 ++- web_app.py | 504 +++++++++++++++++++++++++----------- 25 files changed, 1078 insertions(+), 241 deletions(-) create mode 100644 backend_io.py create mode 100644 job_manager.py create mode 100644 runtime_modules.txt create mode 100644 state_store.py create mode 100644 test_backend_hardening.py diff --git a/Makefile b/Makefile index 6867f08..08f7d04 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,7 @@ DESKTOPDIR = $(PREFIX)/share/applications METAINFODIR = $(PREFIX)/share/metainfo LICENSEDIR = $(PREFIX)/share/licenses/openbox -PYTHON_SOURCES = openbox.py web_app.py openbox_logging.py importers.py arcade.py catalog.py cloud_sync.py \ - emulators.py retroachievements.py plugins.py plugin_runner.py metadata.py \ - archives.py saves.py updates.py env_config.py parity_discovery.py parity_import.py \ - parity_integrations.py parity_media.py parity_saves.py parity_storefront.py plugin_catalog.py parity_premium.py stock_themes.py parity_gameyfin.py parity_save_tools.py \ - parity_filter_presets.py parity_deeplinks.py parity_backup.py parity_tracking.py parity_igdb.py parity_emulator_defs.py parity_import_policy.py parity_gamescope.py +PYTHON_SOURCES = $(shell sed '/^[[:space:]]*#/d;/^[[:space:]]*$$/d' runtime_modules.txt) DATA_FILES = index.html openbox.svg openbox.metainfo.xml LICENSE diff --git a/archives.py b/archives.py index 0a58f77..50397b3 100644 --- a/archives.py +++ b/archives.py @@ -20,9 +20,29 @@ def safe_zip_extract(archive, destination): target = (destination / info.filename).resolve() if root != target and root not in target.parents: raise ValueError(f"Unsafe archive path: {info.filename}") + mode = (info.external_attr >> 16) & 0o170000 + if mode == 0o120000: + raise ValueError(f"Archive symlinks are not supported: {info.filename}") package.extractall(destination) +def validate_7z_paths(extractor, archive): + result = subprocess.run( + [extractor, "l", "-slt", str(archive)], + check=True, + capture_output=True, + text=True, + timeout=60, + ) + for line in result.stdout.splitlines(): + if not line.startswith("Path = "): + continue + name = line[7:] + candidate = Path(name) + if candidate.is_absolute() or ".." in candidate.parts: + raise ValueError(f"Unsafe archive path: {name}") + + def choose_game_file(destination, member=""): if member: selected = (destination / member).resolve() @@ -48,6 +68,7 @@ def extract_game(archive_path, cache_root, member=""): extractor = shutil.which("7z") or shutil.which("7zz") if not extractor: raise FileNotFoundError("7z or 7zz is required to extract this archive.") + validate_7z_paths(extractor, archive) subprocess.run([extractor, "x", "-y", f"-o{destination}", str(archive)], check=True, capture_output=True) complete.touch() return choose_game_file(destination, member) diff --git a/backend_io.py b/backend_io.py new file mode 100644 index 0000000..462bc1b --- /dev/null +++ b/backend_io.py @@ -0,0 +1,97 @@ +"""Shared bounded network and filesystem helpers used by backend operations.""" + +from __future__ import annotations + +import hashlib +import os +import shutil +import tempfile +from pathlib import Path +from urllib.request import Request, urlopen + + +DEFAULT_MAX_DOWNLOAD = 64 * 1024 * 1024 +CHUNK_SIZE = 1024 * 1024 + + +def contained_path(path: Path, roots, *, must_exist=False) -> Path: + candidate = Path(path).expanduser().resolve(strict=False) + allowed = [Path(root).expanduser().resolve(strict=False) for root in roots] + if not any(candidate == root or root in candidate.parents for root in allowed): + raise ValueError(f"Path is outside an approved OpenBox directory: {candidate}") + if must_exist and not candidate.exists(): + raise FileNotFoundError(str(candidate)) + return candidate + + +def safe_media_path(path: Path, data_root: Path) -> Path: + return contained_path(path, [Path(data_root)]) + + +def remove_file_if_safe(path: Path, data_root: Path) -> bool: + target = safe_media_path(path, data_root) + if not target.is_file(): + return False + target.unlink() + return True + + +def download_file( + url: str, + destination: Path, + *, + expected_types=(), + max_bytes=DEFAULT_MAX_DOWNLOAD, + timeout=30, + opener=urlopen, + headers=None, + sha256="", +) -> Path: + request = Request(url, headers={"User-Agent": "OpenBox/1", **(headers or {})}) + destination = Path(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_name = None + digest = hashlib.sha256() + try: + with opener(request, timeout=timeout) as response: + if response.headers and hasattr(response.headers, "get_content_type"): + content_type = response.headers.get_content_type() + else: + content_type = str(response.headers.get("Content-Type", "")).split(";", 1)[0] if response.headers else "" + if expected_types and not any(content_type.startswith(item) for item in expected_types): + raise ValueError(f"The remote server returned an unsupported content type: {content_type or 'unknown'}") + try: + declared = int(response.headers.get("Content-Length", "0")) + except (TypeError, ValueError): + declared = 0 + if declared > max_bytes: + raise ValueError("The download is too large.") + fd, temporary_name = tempfile.mkstemp(prefix=f".{destination.name}.", suffix=".tmp", dir=destination.parent) + with os.fdopen(fd, "wb") as output: + total = 0 + while True: + chunk = response.read(CHUNK_SIZE) + if not chunk: + break + total += len(chunk) + if total > max_bytes: + raise ValueError("The download is too large.") + digest.update(chunk) + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + if sha256 and digest.hexdigest().casefold() != str(sha256).casefold(): + raise ValueError("The downloaded file failed checksum verification.") + os.replace(temporary_name, destination) + temporary_name = None + finally: + if temporary_name: + Path(temporary_name).unlink(missing_ok=True) + return destination + + +def safe_copytree(source: Path, destination: Path) -> None: + source = Path(source) + destination = Path(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(source, destination) diff --git a/build_appimage.sh b/build_appimage.sh index e7ad744..145e51b 100755 --- a/build_appimage.sh +++ b/build_appimage.sh @@ -17,9 +17,11 @@ cp "$python_binary" "$appdir/usr/bin/python3" cp -a "$stdlib" "$appdir/usr/lib/python$python_version" find "$appdir/usr/lib/python$python_version" -type d -name __pycache__ -prune -exec rm -rf -- {} + -for file in openbox.py web_app.py importers.py arcade.py catalog.py cloud_sync.py emulators.py retroachievements.py plugins.py plugin_runner.py metadata.py archives.py saves.py updates.py env_config.py parity_discovery.py parity_import.py parity_integrations.py parity_media.py parity_saves.py parity_storefront.py plugin_catalog.py parity_premium.py stock_themes.py parity_gameyfin.py parity_save_tools.py parity_filter_presets.py parity_deeplinks.py parity_backup.py parity_tracking.py parity_igdb.py parity_emulator_defs.py parity_import_policy.py parity_gamescope.py index.html; do +while IFS= read -r file; do + [ -n "$file" ] || continue cp "$source_root/$file" "$appdir/usr/share/openbox/$file" -done +done < "$source_root/runtime_modules.txt" +cp "$source_root/index.html" "$appdir/usr/share/openbox/index.html" mkdir -p "$appdir/usr/share/openbox/emulator_defs" cp "$source_root"/emulator_defs/*.yaml "$appdir/usr/share/openbox/emulator_defs/" install -Dm755 "$source_root/scripts/openbox-launcher.sh" "$appdir/usr/share/openbox/openbox-launcher.sh" diff --git a/catalog.py b/catalog.py index 170c1e1..0d24e5c 100644 --- a/catalog.py +++ b/catalog.py @@ -56,7 +56,11 @@ def bulk_update(games, ids, changes): clean[field] = {str(key).strip(): str(val).strip() for key, val in value.items() if str(key).strip()} else: clean[field] = str(value).strip() - selected = sorted(set(int(index) for index in ids)) + stable_indexes = {str(game.get("game_id")): index for index, game in enumerate(games) if game.get("game_id")} + selected = sorted({ + stable_indexes[str(value)] if str(value) in stable_indexes else int(value) + for value in ids + }) if selected[0] < 0 or selected[-1] >= len(games): raise IndexError("A selected game no longer exists.") for index in selected: diff --git a/cloud_sync.py b/cloud_sync.py index ea8eca2..87ae3ad 100644 --- a/cloud_sync.py +++ b/cloud_sync.py @@ -16,6 +16,12 @@ def nonnegative_int(value): def game_key(game): + if game.get("game_id"): + return f"id:{game['game_id']}" + return legacy_game_key(game) + + +def legacy_game_key(game): if game.get("steam_app_id"): return f"steam:{game['steam_app_id']}" if game.get("heroic_app_id"): @@ -42,6 +48,8 @@ def sync_statistics(state, folder, now=None): changed = 0 for game in state["games"]: saved = remote_games.get(game_key(game), {}) + if not saved: + saved = remote_games.get(legacy_game_key(game), {}) if not isinstance(saved, dict): continue before = {field:game.get(field) for field in STAT_FIELDS} diff --git a/io.openbox.GameLauncher.yml b/io.openbox.GameLauncher.yml index a84adcb..535587c 100644 --- a/io.openbox.GameLauncher.yml +++ b/io.openbox.GameLauncher.yml @@ -20,7 +20,8 @@ modules: buildsystem: simple build-commands: - mkdir -p /app/share/openbox - - for f in openbox.py web_app.py openbox_logging.py importers.py arcade.py catalog.py cloud_sync.py emulators.py retroachievements.py plugins.py plugin_runner.py metadata.py archives.py saves.py updates.py env_config.py parity_discovery.py parity_import.py parity_integrations.py parity_media.py parity_saves.py parity_storefront.py plugin_catalog.py parity_premium.py stock_themes.py parity_gameyfin.py parity_save_tools.py parity_filter_presets.py parity_deeplinks.py parity_backup.py parity_tracking.py parity_igdb.py parity_emulator_defs.py parity_import_policy.py parity_gamescope.py index.html; do install -Dm644 "$f" "/app/share/openbox/$f"; done + - while IFS= read -r f; do [ -n "$f" ] || continue; install -Dm644 "$f" "/app/share/openbox/$f"; done < runtime_modules.txt + - install -Dm644 index.html /app/share/openbox/index.html - install -d /app/share/openbox/emulator_defs - for f in emulator_defs/*.yaml; do install -Dm644 "$f" "/app/share/openbox/emulator_defs/"; done - install -Dm755 scripts/openbox-launcher.sh /app/share/openbox/openbox-launcher.sh diff --git a/job_manager.py b/job_manager.py new file mode 100644 index 0000000..41c59d7 --- /dev/null +++ b/job_manager.py @@ -0,0 +1,70 @@ +"""Small standard-library job manager for bounded backend work.""" + +from __future__ import annotations + +import logging +import threading +import time +from datetime import datetime, timezone +from typing import Callable + + +LOGGER = logging.getLogger("openbox.jobs") + + +class JobManager: + def __init__(self): + self._lock = threading.RLock() + self._jobs = {} + + def snapshot(self, name): + with self._lock: + return dict(self._jobs.get(name, {})) + + def submit(self, name, worker: Callable[[], dict], *, replace=False): + with self._lock: + current = self._jobs.get(name, {}) + if current.get("state") in {"queued", "running"} and not replace: + return dict(current) + now = datetime.now(timezone.utc).isoformat() + job = { + "name": name, + "state": "queued", + "started_at": "", + "finished_at": "", + "created_at": now, + "error": "", + "attempt": int(current.get("attempt", 0)) + 1, + } + self._jobs[name] = job + + def run(): + with self._lock: + self._jobs[name].update({ + "state": "running", + "started_at": datetime.now(timezone.utc).isoformat(), + }) + started = time.monotonic() + try: + result = worker() or {} + if not isinstance(result, dict): + result = {"result": result} + with self._lock: + self._jobs[name].update(result) + self._jobs[name].update({ + "state": "done", + "finished_at": datetime.now(timezone.utc).isoformat(), + "duration_seconds": round(time.monotonic() - started, 3), + }) + except Exception as error: + LOGGER.exception("Backend job %s failed", name) + with self._lock: + self._jobs[name].update({ + "state": "error", + "error": str(error), + "finished_at": datetime.now(timezone.utc).isoformat(), + "duration_seconds": round(time.monotonic() - started, 3), + }) + + threading.Thread(target=run, name=f"openbox-job-{name}", daemon=True).start() + return dict(job) diff --git a/metadata.py b/metadata.py index f5b3fad..90c3b44 100644 --- a/metadata.py +++ b/metadata.py @@ -9,6 +9,8 @@ from urllib.request import Request, urlopen from xml.etree import ElementTree +from backend_io import download_file + DATABASE_URL = "https://gamesdb.launchbox-app.com/Metadata.zip" IMAGE_URL = "https://images.launchbox-app.com/" @@ -82,11 +84,15 @@ def build_database(metadata_zip, destination): def sync_database(destination, opener=urlopen): destination = Path(destination) destination.parent.mkdir(parents=True, exist_ok=True) - request = Request(DATABASE_URL, headers={"User-Agent":"OpenBox/1"}) with tempfile.NamedTemporaryFile(dir=destination.parent, suffix=".zip", delete=False) as temporary: archive = Path(temporary.name) - with opener(request, timeout=120) as response: - shutil.copyfileobj(response, temporary) + download_file( + DATABASE_URL, + archive, + max_bytes=2 * 1024 * 1024 * 1024, + timeout=120, + opener=opener, + ) try: build_database(archive, destination) finally: @@ -110,16 +116,14 @@ def search_games(database_path, title, platform="", limit=20): def download_image(filename, destination, opener=urlopen): filename = Path(filename).name - request = Request(IMAGE_URL + filename, headers={"User-Agent":"OpenBox/1"}) - with opener(request, timeout=30) as response: - if not response.headers.get_content_type().startswith("image/"): - raise ValueError("The metadata server did not return an image.") - destination.parent.mkdir(parents=True, exist_ok=True) - temporary = destination.with_suffix(destination.suffix + ".tmp") - with temporary.open("wb") as output: - shutil.copyfileobj(response, output) - temporary.replace(destination) - return str(destination) + return str(download_file( + IMAGE_URL + filename, + destination, + expected_types=("image/",), + max_bytes=32 * 1024 * 1024, + timeout=30, + opener=opener, + )) def apply_game_metadata(game, database_path, database_id, media_types, media_root, overwrite=False, opener=urlopen, region_priority=None): diff --git a/openbox.py b/openbox.py index 165beb9..bb77dea 100644 --- a/openbox.py +++ b/openbox.py @@ -17,6 +17,7 @@ from archives import extract_game from openbox_logging import configure_logging +from state_store import JsonStateStore CUSTOM_DATA_DIR = os.environ.get("OPENBOX_DATA_DIR") APP_DIR = Path(CUSTOM_DATA_DIR or Path.home() / ".local/share/openbox-game-launcher").expanduser() @@ -27,6 +28,8 @@ shutil.copy2(LEGACY_DATA, DATA) from parity_import import EXTENSIONS_EXTRA, PLATFORM_BY_EXTENSION_EXTRA +STATE_STORE = JsonStateStore(DATA) + EXTENSIONS = {".sh", ".appimage", ".exe", ".iso", ".rom", ".nes", ".sfc", ".smc", ".gba", ".gb", ".gbc", ".zip", ".7z", ".rar"} | EXTENSIONS_EXTRA PLATFORM_BY_EXTENSION = { ".nes": "NES", ".sfc": "SNES", ".smc": "SNES", ".gba": "Game Boy Advance", @@ -59,27 +62,20 @@ def purge_demo_games(state): def load_state(): - try: - raw = json.loads(DATA.read_text()) - if isinstance(raw, list): - return {"games": raw, "profiles": {}, "history": []} - if not isinstance(raw, dict): - raise AttributeError - raw.setdefault("games", []) - raw.setdefault("profiles", {}) - raw.setdefault("history", []) - raw.setdefault("settings", {}) - raw.setdefault("playlists", []) - return raw - except (FileNotFoundError, json.JSONDecodeError, AttributeError): - return {"games": [], "profiles": {}, "history": [], "settings": {}, "playlists": []} + return STATE_STORE.load() def save_state(state): - DATA.parent.mkdir(parents=True, exist_ok=True) - temporary = DATA.with_suffix(".tmp") - temporary.write_text(json.dumps(state, indent=2)) - temporary.replace(DATA) + return STATE_STORE.save(state) + + +def update_state(mutator): + """Apply one state mutation under the cross-process transaction lock.""" + return STATE_STORE.update(mutator) + + +def recover_state(): + return STATE_STORE.recover() def format_duration(seconds): diff --git a/parity_gameyfin.py b/parity_gameyfin.py index e570fa3..8348e89 100644 --- a/parity_gameyfin.py +++ b/parity_gameyfin.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os import re import shutil import urllib.error @@ -73,7 +74,8 @@ def request(self, method, path, data=None, headers=None, raw=False): raise GameyfinError(f"Gameyfin request failed ({error.code}): {detail or error.reason}") from error except urllib.error.URLError as error: raise GameyfinError(f"Could not reach Gameyfin: {error.reason}") from error - payload = response.read() + with response: + payload = response.read() if raw: return response, payload content_type = response.headers.get("Content-Type", "") @@ -163,12 +165,18 @@ def download_game(self, game_id, provider, destination): target = destination / filename partial = destination / f".{filename}.partial" try: - with partial.open("wb") as handle: + with response, partial.open("wb") as handle: + total = 0 while True: chunk = response.read(1024 * 1024) if not chunk: break + total += len(chunk) + if total > 4 * 1024 * 1024 * 1024: + raise GameyfinError("The Gameyfin download is too large.") handle.write(chunk) + handle.flush() + os.fsync(handle.fileno()) partial.replace(target) except Exception: if partial.exists(): @@ -306,8 +314,14 @@ def install_gameyfin_game(settings, game_id, client=None): def uninstall_gameyfin_game(game): install_dir = Path(str(game.get("install_dir") or "")).expanduser() path = Path(str(game.get("path") or "")).expanduser() + if not str(game.get("install_dir") or "").strip(): + raise GameyfinError("Refusing to uninstall a Gameyfin game without an install directory.") + root = install_dir.resolve(strict=False) removed = [] for candidate in (install_dir, path): + candidate = candidate.resolve(strict=False) + if candidate != root and root not in candidate.parents: + raise GameyfinError(f"Refusing to remove a path outside the Gameyfin install directory: {candidate}") if not candidate.exists(): continue if candidate.is_dir(): diff --git a/parity_integrations.py b/parity_integrations.py index a4e1251..939d4bb 100644 --- a/parity_integrations.py +++ b/parity_integrations.py @@ -13,6 +13,9 @@ from pathlib import Path from urllib.request import Request, urlopen +from archives import safe_zip_extract +from backend_io import download_file + PLATFORM_BEZEL_REPO = { "NES": "thebezelproject/bezelproject-NintendoEntertainmentSystem", @@ -117,14 +120,17 @@ def download_bezel(platform, dest_dir, opener=urlopen): dest.mkdir(parents=True, exist_ok=True) archive = dest / f"{platform.replace(' ', '_')}-bezels.zip" request = Request(urls[platform], headers={"User-Agent": "OpenBox/1"}) - with opener(request, timeout=120) as response, archive.open("wb") as output: - shutil.copyfileobj(response, output) + download_file( + urls[platform], archive, + max_bytes=512 * 1024 * 1024, + timeout=120, + opener=opener, + ) extract_to = dest / platform.replace(" ", "_") if extract_to.exists(): shutil.rmtree(extract_to) extract_to.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(archive) as package: - package.extractall(extract_to) + safe_zip_extract(archive, extract_to) return str(extract_to) @@ -158,22 +164,23 @@ def download_emumovies_media(game, credentials, media_root, media_type="box"): platform = str(game.get("platform") or "Arcade") name = str(game.get("name") or "") url = f"https://api.emumovies.com/v1/media/{media_type}?system={platform}&search={name}" - request = Request(url, headers={"User-Agent": "OpenBox/1"}) import base64 token = base64.b64encode(f"{username}:{password}".encode()).decode() - request.add_header("Authorization", f"Basic {token}") - try: - with urlopen(request, timeout=30) as response: - payload = response.read() - content_type = response.headers.get_content_type() - except Exception as error: # noqa: BLE001 - surface remote/API failures cleanly - raise ValueError(f"EmuMovies request failed: {error}") from error - if not content_type.startswith("image/"): - raise ValueError("EmuMovies did not return image media for this title.") root = Path(media_root) / "emumovies" / re.sub(r"[^a-z0-9]+", "-", name.casefold()).strip("-") root.mkdir(parents=True, exist_ok=True) destination = root / f"{media_type}.jpg" - destination.write_bytes(payload) + try: + download_file( + url, + destination, + expected_types=("image/",), + max_bytes=32 * 1024 * 1024, + timeout=30, + opener=urlopen, + headers={"Authorization": f"Basic {token}"}, + ) + except Exception as error: # noqa: BLE001 - surface remote/API failures cleanly + raise ValueError(f"EmuMovies request failed: {error}") from error return str(destination) diff --git a/parity_media.py b/parity_media.py index bfb1df0..de495fd 100644 --- a/parity_media.py +++ b/parity_media.py @@ -91,16 +91,20 @@ def find_duplicate_media(games): return groups -def cleanup_duplicates(duplicate_groups, dry_run=True): +def cleanup_duplicates(duplicate_groups, dry_run=True, allowed_roots=None): deleted = [] + roots = [Path(root).expanduser().resolve(strict=False) for root in (allowed_roots or [])] for group in duplicate_groups: for path in group.get("duplicates", []): target = Path(path) if dry_run: deleted.append(str(target)) continue + resolved = target.expanduser().resolve(strict=False) + if roots and not any(resolved == root or root in resolved.parents for root in roots): + continue try: - target.unlink(missing_ok=True) + resolved.unlink(missing_ok=True) deleted.append(str(target)) except OSError: pass diff --git a/parity_premium.py b/parity_premium.py index 608e47b..0a56b5c 100644 --- a/parity_premium.py +++ b/parity_premium.py @@ -12,6 +12,8 @@ from urllib.parse import quote from urllib.request import Request, urlopen +from backend_io import download_file + try: import py7zr except ImportError: @@ -245,11 +247,13 @@ def apply_esrb_from_record(game, record): def download_bytes(url, destination, opener=urlopen): - request = Request(url, headers={"User-Agent": "OpenBox/1"}) - with opener(request, timeout=20) as response: - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(response.read()) - return str(destination) + return str(download_file( + url, + destination, + max_bytes=512 * 1024 * 1024, + timeout=20, + opener=opener, + )) def download_steam_trailer(game, media_root, opener=urlopen): diff --git a/parity_saves.py b/parity_saves.py index e676f3d..ece0c19 100644 --- a/parity_saves.py +++ b/parity_saves.py @@ -2,11 +2,17 @@ from __future__ import annotations +import threading +import time from pathlib import Path from saves import discover_save_paths, list_backups +_SAVE_CACHE_LOCK = threading.RLock() +_SAVE_CACHE = {"at": 0.0, "signature": None, "indices": []} + + def extra_save_candidates(game, home=None): home = Path(home or Path.home()) platform = str(game.get("platform", "")) @@ -54,9 +60,6 @@ def scan_all_saves(games, home=None): for item in extra_save_candidates(game, home=home): if item["path"] not in paths: paths.append(item["path"]) - for item in extra_save_candidates(game, home=home): - if item["path"] not in paths: - paths.append(item["path"]) existing = [path for path in paths if Path(path).exists()] if existing: found[index] = existing @@ -64,9 +67,24 @@ def scan_all_saves(games, home=None): def games_with_saves(games, home=None): + home_key = str(Path(home or Path.home()).expanduser()) + signature = tuple( + ( + str(game.get("game_id") or index), + str(game.get("path") or ""), + tuple(str(path) for path in game.get("save_paths", []) if str(path).strip()), + ) + for index, game in enumerate(games) + ) + (home_key,) + with _SAVE_CACHE_LOCK: + if _SAVE_CACHE["signature"] == signature and time.monotonic() - _SAVE_CACHE["at"] < 2: + return list(_SAVE_CACHE["indices"]) scanned = scan_all_saves(games, home=home) indices = set(scanned) for index, game in enumerate(games): if any(Path(path).expanduser().exists() for path in game.get("save_paths", []) if str(path).strip()): indices.add(index) - return sorted(indices) + result = sorted(indices) + with _SAVE_CACHE_LOCK: + _SAVE_CACHE.update({"at": time.monotonic(), "signature": signature, "indices": result}) + return result diff --git a/plugin_catalog.py b/plugin_catalog.py index c4830a9..fbe152d 100644 --- a/plugin_catalog.py +++ b/plugin_catalog.py @@ -6,6 +6,8 @@ from pathlib import Path from urllib.request import Request, urlopen +from backend_io import download_file + CATALOG_PATH = Path(__file__).resolve().parent / "plugins" / "catalog.json" REMOTE_CATALOG = "https://raw.githubusercontent.com/vindeckyy/OpenBoxGL/master/plugins/catalog.json" @@ -39,7 +41,12 @@ def download_plugin_package(entry, dest_dir, opener=urlopen): dest = Path(dest_dir) dest.mkdir(parents=True, exist_ok=True) archive = dest / f"{entry.get('id', 'plugin')}.zip" - request = Request(url, headers={"User-Agent": "OpenBox/1"}) - with opener(request, timeout=120) as response, archive.open("wb") as output: - output.write(response.read()) + download_file( + url, + archive, + max_bytes=128 * 1024 * 1024, + timeout=120, + opener=opener, + sha256=str(entry.get("sha256") or "").strip(), + ) return archive diff --git a/plugins.py b/plugins.py index e9d777b..a5cb9fc 100644 --- a/plugins.py +++ b/plugins.py @@ -1,6 +1,7 @@ """Local OpenBox plugin packages and hooks.""" import json +import logging import re import shutil import subprocess @@ -15,6 +16,8 @@ HOOKS = {"before_launch", "after_session", "library"} PLUGIN_ID = re.compile(r"^[a-z0-9][a-z0-9._-]{1,63}$") RUNNER = Path(__file__).with_name("plugin_runner.py") +LOGGER = logging.getLogger("openbox.plugins") +MAX_PLUGIN_PAYLOAD = 2 * 1024 * 1024 def state_file(directory): @@ -124,12 +127,20 @@ def run_plugins(directory, hook, payload): if not manifest["enabled"] or hook not in manifest["hooks"]: continue entry = Path(directory) / manifest["id"] / manifest["entry"] + encoded = json.dumps(result) + if len(encoded.encode("utf-8")) > MAX_PLUGIN_PAYLOAD: + LOGGER.warning("Skipping plugin %s for %s because the payload is too large", manifest["id"], hook) + continue try: completed = subprocess.run( [sys.executable, str(RUNNER), str(entry), hook], - input=json.dumps(result), capture_output=True, text=True, timeout=5, + input=encoded, capture_output=True, text=True, timeout=5, ) - except (OSError, subprocess.SubprocessError): + except (OSError, subprocess.SubprocessError) as error: + LOGGER.warning("Plugin %s failed for %s: %s", manifest["id"], hook, error) + continue + if len(completed.stdout.encode("utf-8")) > MAX_PLUGIN_PAYLOAD: + LOGGER.warning("Ignoring oversized output from plugin %s", manifest["id"]) continue if completed.returncode == 0 and completed.stdout.strip(): try: @@ -137,5 +148,7 @@ def run_plugins(directory, hook, payload): if isinstance(candidate, dict): result = candidate except json.JSONDecodeError: - pass + LOGGER.warning("Ignoring invalid JSON from plugin %s", manifest["id"]) + elif completed.returncode: + LOGGER.warning("Plugin %s exited with status %s: %s", manifest["id"], completed.returncode, completed.stderr[-400:]) return result diff --git a/retroachievements.py b/retroachievements.py index 7a206d0..831ef7d 100644 --- a/retroachievements.py +++ b/retroachievements.py @@ -10,6 +10,8 @@ from urllib.parse import urlencode from urllib.request import Request, urlopen +from state_store import secure_text_write + SYSTEM_NAMES = { "nes": ("Nintendo Entertainment System", "NES/Famicom"), @@ -66,11 +68,7 @@ def save_credentials(directory, username, api_key, fetch=api_get): if not profile.get("User"): raise ValueError("RetroAchievements rejected those credentials.") path = Path(directory) / "retroachievements.json" - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_suffix(".tmp") - temporary.write_text(json.dumps(credentials)) - os.chmod(temporary, 0o600) - temporary.replace(path) + secure_text_write(path, json.dumps(credentials)) return profile diff --git a/runtime_modules.txt b/runtime_modules.txt new file mode 100644 index 0000000..5586f76 --- /dev/null +++ b/runtime_modules.txt @@ -0,0 +1,38 @@ +openbox.py +web_app.py +openbox_logging.py +state_store.py +backend_io.py +job_manager.py +importers.py +arcade.py +catalog.py +cloud_sync.py +emulators.py +retroachievements.py +plugins.py +plugin_runner.py +metadata.py +archives.py +saves.py +updates.py +env_config.py +parity_discovery.py +parity_import.py +parity_integrations.py +parity_media.py +parity_saves.py +parity_storefront.py +plugin_catalog.py +parity_premium.py +stock_themes.py +parity_gameyfin.py +parity_save_tools.py +parity_filter_presets.py +parity_deeplinks.py +parity_backup.py +parity_tracking.py +parity_igdb.py +parity_emulator_defs.py +parity_import_policy.py +parity_gamescope.py diff --git a/saves.py b/saves.py index 8b645da..5f3c252 100644 --- a/saves.py +++ b/saves.py @@ -138,7 +138,10 @@ def restore_saves(game, root, backup_name): if index >= len(roots): raise ValueError("Invalid save backup manifest.") root = roots[index] - destination = (root["path"].parent if root["file"] else root["path"]) / Path(*parts[2:]) + base = (root["path"].parent if root["file"] else root["path"]).resolve() + destination = (base / Path(*parts[2:])).resolve() + if destination != base and base not in destination.parents: + raise ValueError("Save backup contains an unsafe path.") destination.parent.mkdir(parents=True, exist_ok=True) destination.write_bytes(package.read(info)) return archive diff --git a/state_store.py b/state_store.py new file mode 100644 index 0000000..2dc2ca9 --- /dev/null +++ b/state_store.py @@ -0,0 +1,210 @@ +"""Transactional, process-safe JSON persistence for OpenBox user data.""" + +from __future__ import annotations + +import copy +import fcntl +import json +import os +import shutil +import tempfile +import threading +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Callable + + +STATE_SCHEMA_VERSION = 2 + + +class StateCorruptError(RuntimeError): + """Raised when the primary state file cannot be decoded safely.""" + + +def default_state() -> dict[str, Any]: + return { + "schema_version": STATE_SCHEMA_VERSION, + "games": [], + "profiles": {}, + "history": [], + "settings": {}, + "playlists": [], + } + + +def _stable_game_id(game: dict[str, Any], index: int) -> str: + """Return a durable id for a game, including legacy records without one.""" + for key in ("game_id", "id"): + value = str(game.get(key) or "").strip() + if value and key == "game_id": + return value + identity = { + key: str(game.get(key) or "").strip() + for key in ( + "path", "name", "platform", "steam_app_id", "heroic_app_id", + "lutris_id", "gameyfin_id", "launchbox_db_id", + ) + } + raw = json.dumps(identity, sort_keys=True, separators=(",", ":")) + import hashlib + + digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24] + return f"game-{digest}-{index}" + + +def normalize_state(raw: Any) -> tuple[dict[str, Any], bool]: + """Normalize legacy state while retaining every unknown field.""" + changed = False + if isinstance(raw, list): + state: dict[str, Any] = {"games": raw} + changed = True + elif isinstance(raw, dict): + state = copy.deepcopy(raw) + else: + raise StateCorruptError("OpenBox library.json must contain an object or legacy game list.") + + defaults = default_state() + for key, value in defaults.items(): + if key not in state: + state[key] = copy.deepcopy(value) + changed = True + + try: + version = int(state.get("schema_version", 1)) + except (TypeError, ValueError): + version = 1 + changed = True + if version > STATE_SCHEMA_VERSION: + raise StateCorruptError( + f"OpenBox library.json uses unsupported schema version {version}." + ) + if version != STATE_SCHEMA_VERSION: + state["schema_version"] = STATE_SCHEMA_VERSION + changed = True + + if not isinstance(state["games"], list): + raise StateCorruptError("OpenBox library.json has an invalid games collection.") + for index, game in enumerate(state["games"]): + if not isinstance(game, dict): + raise StateCorruptError(f"OpenBox library.json has an invalid game at index {index}.") + game_id = _stable_game_id(game, index) + if game.get("game_id") != game_id: + game["game_id"] = game_id + changed = True + return state, changed + + +class JsonStateStore: + """A small JSON store with atomic commits and a sidecar last-known-good copy.""" + + def __init__(self, path: Path): + self.path = Path(path) + self.lock_path = self.path.with_name(f".{self.path.name}.lock") + self.backup_path = self.path.with_name(f"{self.path.name}.bak") + self._thread_lock = threading.RLock() + + @contextmanager + def _file_lock(self, exclusive: bool): + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.lock_path.open("a+", encoding="utf-8") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + def _read_unlocked(self, path: Path) -> Any: + with path.open("r", encoding="utf-8") as source: + return json.load(source) + + def _load_unlocked(self) -> tuple[dict[str, Any], bool]: + if not self.path.exists(): + return default_state(), False + try: + raw = self._read_unlocked(self.path) + except (OSError, json.JSONDecodeError, UnicodeDecodeError) as error: + raise StateCorruptError( + f"Unable to read {self.path}. The original file was preserved; restore or inspect it before continuing." + ) from error + return normalize_state(raw) + + def load(self) -> dict[str, Any]: + with self._thread_lock, self._file_lock(True): + state, changed = self._load_unlocked() + if changed: + self._write_unlocked(state) + return state + + def recover(self) -> dict[str, Any]: + with self._thread_lock, self._file_lock(True): + if not self.backup_path.is_file(): + raise StateCorruptError(f"No last-known-good state exists at {self.backup_path}.") + try: + state, _ = normalize_state(self._read_unlocked(self.backup_path)) + except (OSError, json.JSONDecodeError, UnicodeDecodeError, StateCorruptError) as error: + raise StateCorruptError(f"The last-known-good state is also unusable: {self.backup_path}") from error + self._write_unlocked(state) + return state + + def _write_unlocked(self, state: dict[str, Any]) -> None: + normalized, _ = normalize_state(state) + self.path.parent.mkdir(parents=True, exist_ok=True) + if self.path.is_file(): + shutil.copy2(self.path, self.backup_path) + os.chmod(self.backup_path, 0o600) + fd, temporary_name = tempfile.mkstemp( + prefix=f".{self.path.name}.", suffix=".tmp", dir=self.path.parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as output: + json.dump(normalized, output, indent=2, ensure_ascii=False) + output.write("\n") + output.flush() + os.fsync(output.fileno()) + os.chmod(temporary, 0o600) + os.replace(temporary, self.path) + os.chmod(self.path, 0o600) + shutil.copy2(self.path, self.backup_path) + os.chmod(self.backup_path, 0o600) + directory_fd = os.open(self.path.parent, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + if temporary.exists(): + temporary.unlink() + + def save(self, state: dict[str, Any]) -> dict[str, Any]: + with self._thread_lock, self._file_lock(True): + normalized, _ = normalize_state(state) + self._write_unlocked(normalized) + return normalized + + def update(self, mutator: Callable[[dict[str, Any]], Any]) -> dict[str, Any]: + with self._thread_lock, self._file_lock(True): + state, _ = self._load_unlocked() + mutator(state) + normalized, _ = normalize_state(state) + self._write_unlocked(normalized) + return normalized + + +def secure_text_write(path: Path, value: str) -> None: + """Write a credential or token file atomically with owner-only permissions.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) + temporary = Path(temporary_name) + try: + os.chmod(temporary, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as output: + output.write(value) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, path) + os.chmod(path, 0o600) + finally: + if temporary.exists(): + temporary.unlink() diff --git a/test_backend_hardening.py b/test_backend_hardening.py new file mode 100644 index 0000000..c99a4a0 --- /dev/null +++ b/test_backend_hardening.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +import hashlib +import io +import json +import os +import stat +import tempfile +import unittest +import zipfile +from pathlib import Path + +from backend_io import download_file +from catalog import bulk_update +from parity_gameyfin import GameyfinError, uninstall_gameyfin_game +from state_store import JsonStateStore, StateCorruptError, STATE_SCHEMA_VERSION + + +class FakeResponse(io.BytesIO): + def __init__(self, payload, headers=None): + super().__init__(payload) + self.headers = headers or {"Content-Type": "application/octet-stream", "Content-Length": str(len(payload))} + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + +class BackendHardeningTests(unittest.TestCase): + def test_state_migration_ids_and_recovery(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "library.json" + store = JsonStateStore(path) + store.save({"games": [{"name": "Legacy", "path": "/tmp/legacy.rom"}], "profiles": {}, "history": []}) + state = store.load() + self.assertEqual(state["schema_version"], STATE_SCHEMA_VERSION) + stable_id = state["games"][0]["game_id"] + self.assertTrue(stable_id) + store.save({**state, "games": [{**state["games"][0], "favorite": True}]}) + path.write_text("{broken", encoding="utf-8") + with self.assertRaises(StateCorruptError): + store.load() + recovered = store.recover() + self.assertEqual(recovered["games"][0]["game_id"], stable_id) + self.assertTrue(recovered["games"][0]["favorite"]) + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600) + + def test_stable_ids_are_accepted_by_bulk_updates(self): + games = [{"game_id": "game-a", "name": "A"}, {"game_id": "game-b", "name": "B"}] + self.assertEqual(bulk_update(games, ["game-b"], {"favorite": True}), 1) + self.assertTrue(games[1]["favorite"]) + + def test_bounded_atomic_download_and_checksum(self): + with tempfile.TemporaryDirectory() as directory: + destination = Path(directory) / "payload.bin" + payload = b"safe payload" + digest = hashlib.sha256(payload).hexdigest() + result = download_file( + "https://example.invalid/payload", + destination, + opener=lambda *_args, **_kwargs: FakeResponse(payload), + sha256=digest, + max_bytes=1024, + ) + self.assertEqual(result.read_bytes(), payload) + with self.assertRaises(ValueError): + download_file( + "https://example.invalid/payload", + destination, + opener=lambda *_args, **_kwargs: FakeResponse(payload), + sha256="0" * 64, + max_bytes=1024, + ) + + def test_gameyfin_uninstall_rejects_outside_path(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "install" + root.mkdir() + outside = Path(directory) / "outside.dat" + outside.write_text("keep", encoding="utf-8") + with self.assertRaises(GameyfinError): + uninstall_gameyfin_game({"install_dir": str(root), "path": str(outside)}) + self.assertTrue(outside.exists()) + + def test_archive_symlink_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "link.zip" + info = zipfile.ZipInfo("link") + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + with zipfile.ZipFile(archive, "w") as package: + package.writestr(info, "/etc/passwd") + from archives import safe_zip_extract + + with self.assertRaises(ValueError): + safe_zip_extract(archive, Path(directory) / "out") + + +if __name__ == "__main__": + unittest.main() diff --git a/test_bug_sweep_api.py b/test_bug_sweep_api.py index afcb25e..f2b34e1 100644 --- a/test_bug_sweep_api.py +++ b/test_bug_sweep_api.py @@ -82,6 +82,7 @@ def test_fixture(self): status, payload = self.request("/api/library") self.assertEqual(status, 200) self.assertEqual(payload["games"][0]["name"], "Fixture") + self.assertTrue(payload["games"][0]["game_id"]) self.assertTrue(str(self.web_app.DATA).startswith(self.tempdir.name)) self.assert_alive() diff --git a/test_packaging.py b/test_packaging.py index fe12726..48c93a2 100644 --- a/test_packaging.py +++ b/test_packaging.py @@ -8,16 +8,7 @@ ROOT = Path(__file__).parent -PYTHON_MODULES = [ - "openbox.py", "web_app.py", "openbox_logging.py", "importers.py", "arcade.py", "catalog.py", - "cloud_sync.py", "emulators.py", "retroachievements.py", "plugins.py", - "plugin_runner.py", "metadata.py", "archives.py", "saves.py", "updates.py", - "env_config.py", "parity_discovery.py", "parity_import.py", "parity_integrations.py", - "parity_media.py", "parity_saves.py", "parity_storefront.py", "plugin_catalog.py", "parity_premium.py", - "stock_themes.py", "parity_gameyfin.py", "parity_save_tools.py", - "parity_filter_presets.py", "parity_deeplinks.py", "parity_backup.py", "parity_tracking.py", - "parity_igdb.py", "parity_emulator_defs.py", "parity_import_policy.py", "parity_gamescope.py", -] +PYTHON_MODULES = [line.strip() for line in (ROOT / "runtime_modules.txt").read_text().splitlines() if line.strip() and not line.lstrip().startswith("#")] DATA_FILES = ["index.html"] STOCK_THEMES = [ "Midnight Circuit.css", @@ -34,6 +25,11 @@ def test_appdir_structure(): if not appimage.exists(): print(" skipping AppImage test (not built)") return + if not appimage_path: + source_mtime = max((ROOT / module).stat().st_mtime for module in PYTHON_MODULES) + if appimage.stat().st_mtime < source_mtime: + print(" skipping stale bundled AppImage; set OPENBOX_APPIMAGE to validate a rebuilt artifact") + return appdir = ROOT / "squashfs-root" try: subprocess.run( @@ -46,9 +42,7 @@ def test_appdir_structure(): share = appdir / "usr" / "share" / "openbox" assert share.is_dir(), "missing openbox data dir" missing = [module for module in PYTHON_MODULES if not (share / module).is_file()] - if missing: - print(f" skipping AppImage module check (rebuild needed): {', '.join(missing)}") - return + assert not missing, f"missing runtime modules in AppImage: {', '.join(missing)}" assert (share / "openbox-launcher.sh").is_file(), "missing keyboard launcher" for data in DATA_FILES: assert (share / data).is_file(), f"missing {data} in AppImage" @@ -82,6 +76,19 @@ def test_makefile_install(): print(" Makefile scripts: ok") +def test_runtime_manifest(): + manifest = ROOT / "runtime_modules.txt" + modules = [line.strip() for line in manifest.read_text().splitlines() if line.strip()] + assert len(modules) == len(set(modules)), "runtime module manifest contains duplicates" + missing = [module for module in modules if not (ROOT / module).is_file()] + assert not missing, f"runtime module manifest has missing files: {missing}" + build_script = (ROOT / "build_appimage.sh").read_text() + flatpak = (ROOT / "io.openbox.GameLauncher.yml").read_text() + assert "runtime_modules.txt" in build_script + assert "runtime_modules.txt" in flatpak + print(" Runtime module manifest: ok") + + def test_flatpak_manifest(): manifest = ROOT / "io.openbox.GameLauncher.yml" assert manifest.exists(), "missing Flatpak manifest" @@ -90,7 +97,9 @@ def test_flatpak_manifest(): assert "runtime: org.freedesktop.Platform" in content assert "command: openbox" in content assert "openbox.sh" in content - assert "openbox_logging.py" in content + runtime_modules = (ROOT / "runtime_modules.txt").read_text() + assert "openbox_logging.py" in runtime_modules + assert "runtime_modules.txt" in content assert "openbox.svg" in content print(" Flatpak manifest: ok") @@ -188,6 +197,7 @@ def main(): test_legal_policy() test_flatpak_manifest() test_makefile_install() + test_runtime_manifest() test_appdir_structure() test_version_consistency() test_update_verification() diff --git a/web_app.py b/web_app.py index 7af5f22..72c5345 100644 --- a/web_app.py +++ b/web_app.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Local browser UI for OpenBox. Independent open-source software not affiliated with LaunchBox or Unbroken Software, LLC.""" +import copy import json import html import logging @@ -16,6 +17,7 @@ import sys import tempfile import threading +import time import zipfile from datetime import datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -24,13 +26,16 @@ from urllib.parse import parse_qs, urlparse from arcade import import_arcade +from backend_io import download_file, remove_file_if_safe from catalog import PROGRESS, apply_progress_automation, bulk_update, game_media_paths, related_game_ids from cloud_sync import sync_statistics from emulators import emulator_status, install_all_emulators, install_emulator, launch_emulator, recommendations_for_platform, update_all_emulators, update_emulator from importers import import_heroic, import_lutris, import_steam from metadata import apply_game_metadata, search_games, sync_database +from job_manager import JobManager from openbox_logging import configure_logging, read_diagnostic_log -from openbox import DATA, EXTENSIONS, PLATFORM_BY_EXTENSION, build_launch, discover_profiles, load_state, purge_demo_games, save_state +from openbox import DATA, EXTENSIONS, PLATFORM_BY_EXTENSION, build_launch, discover_profiles, load_state, purge_demo_games, recover_state as recover_library_state, save_state +from state_store import StateCorruptError, secure_text_write from env_config import bootstrap_env from parity_discovery import discovery_lists, related_with_reasons from parity_import import detect_dependencies, import_multi_platform, import_rpcs3_hdd, import_scummvm, import_vita3k, recommend_emulators @@ -119,6 +124,10 @@ INSTALLS = {} METADATA_JOB = {} MEDIA_JOB = {} +JOB_MANAGER = JobManager() +FILE_PROBE_CACHE = {} +FILE_PROBE_LOCK = threading.Lock() +FILE_PROBE_TTL = 1.0 WATCH_STOP = threading.Event() METADATA_DATABASE = DATA.parent / "metadata/launchbox.db" FIELDS = { @@ -134,6 +143,28 @@ } +def probe_path(path, *, file_only=False): + value = str(path or "") + if not value: + return False + now = time.monotonic() + key = (value, file_only) + with FILE_PROBE_LOCK: + cached = FILE_PROBE_CACHE.get(key) + if cached and now - cached[0] < FILE_PROBE_TTL: + return cached[1] + candidate = Path(value) + result = candidate.is_file() if file_only else candidate.exists() + with FILE_PROBE_LOCK: + FILE_PROBE_CACHE[key] = (now, result) + if len(FILE_PROBE_CACHE) > 10000: + cutoff = now - FILE_PROBE_TTL + for cached_key, (created, _) in list(FILE_PROBE_CACHE.items()): + if created < cutoff: + FILE_PROBE_CACHE.pop(cached_key, None) + return result + + def game_identity(game): if game.get("steam_app_id"): return "steam", str(game["steam_app_id"]) @@ -214,8 +245,15 @@ def merge_imported_games(imported, identity_fn): def auto_import_worker(): - while not WATCH_STOP.wait(10): - state = load_state() + delay = 10 + while not WATCH_STOP.wait(delay): + try: + state = load_state() + except Exception as error: + LOGGER.exception("Automatic import paused because library state could not be read: %s", error) + delay = min(delay * 2, 300) + continue + delay = 10 settings = state.get("settings", {}) folders = settings.get("watch_folders", []) for folder in folders: @@ -356,10 +394,11 @@ def public_state(): normalize_video_fields(game) visible = {key: game.get(key, "") for key in FIELDS} video_field, video_path = active_video(game, state.get("settings", {}).get("video_priority")) - path_exists = bool(game.get("path")) and Path(game["path"]).exists() + path_exists = probe_path(game.get("path")) store_installed = bool(game["store_installed"]) if "store_installed" in game else path_exists visible.update({ "id": index, + "game_id": game.get("game_id", ""), "favorite": bool(game.get("favorite")), "hidden": bool(game.get("hidden")), "hide_in_bigbox": bool(game.get("hide_in_bigbox")), @@ -367,11 +406,11 @@ def public_state(): "play_count": game.get("play_count", 0), "playtime_seconds": game.get("playtime_seconds", 0), "path_exists": path_exists, - "has_cover": bool(game.get("cover")) and Path(game["cover"]).is_file(), - "has_background": bool(game.get("background")) and Path(game["background"]).is_file(), + "has_cover": probe_path(game.get("cover"), file_only=True), + "has_background": probe_path(game.get("background"), file_only=True), "has_video": bool(video_path), "active_video_field": video_field, - "has_music": bool(game.get("music")) and Path(game["music"]).is_file(), + "has_music": probe_path(game.get("music"), file_only=True), "has_saves": index in save_indices or bool(game.get("save_paths")), "has_documents": bool(game.get("documents")), "extract_archive": bool(game.get("extract_archive")), @@ -383,7 +422,7 @@ def public_state(): "alternate_names": game.get("alternate_names", []) if isinstance(game.get("alternate_names"), list) else [name for name in str(game.get("alternate_names") or "").split(";") if name.strip()], "available_screenshots": [ index for index, path in enumerate(game.get("screenshots", [])) - if Path(path).is_file() + if probe_path(path, file_only=True) ], "esrb": game.get("esrb", ""), "custom_fields": game.get("custom_fields", {}) if isinstance(game.get("custom_fields"), dict) else {}, @@ -403,6 +442,7 @@ def public_state(): games = decorated for index, game in enumerate(games): game["id"] = index + game.setdefault("game_id", state["games"][index].get("game_id", "")) return { "games": games, "playlists": state.get("playlists", []), @@ -432,6 +472,11 @@ def resolve_library_game(state, identity, fallback_index=None): games = state.get("games") or [] if not isinstance(identity, dict): identity = {} + stable_id = str(identity.get("stable_game_id") or identity.get("game_id") or "").strip() + if stable_id: + for game in games: + if str(game.get("game_id") or "") == stable_id: + return game for key in ("gameyfin_id", "steam_app_id", "heroic_app_id", "lutris_id"): value = str(identity.get(key) or "").strip() if not value: @@ -464,10 +509,40 @@ def resolve_library_game(state, identity, fallback_index=None): return None +def game_from_payload(state, payload): + """Resolve additive stable IDs first, then retain the numeric frontend ID.""" + if not isinstance(payload, dict): + raise ValueError("Request payload must be an object.") + stable_id = str(payload.get("game_id") or payload.get("stable_game_id") or "").strip() + games = state.get("games") or [] + if stable_id: + for game in games: + if str(game.get("game_id") or "") == stable_id: + return game + raise IndexError("Game not found") + if payload.get("id") is None: + raise ValueError("Game id is required.") + try: + index = int(payload["id"]) + except (TypeError, ValueError) as error: + raise ValueError("Game id must be a number or stable game id.") from error + if index < 0 or index >= len(games): + raise IndexError("Game not found") + return games[index] + + +def game_from_query(state, query): + payload = {"id": query.get("id", [None])[0]} + if query.get("game_id", [""])[0]: + payload["game_id"] = query["game_id"][0] + return game_from_payload(state, payload) + + def finish_session(launch_id, game_index, started, process): with PROCESS_LOCK: running_snapshot = dict(RUNNING.get(launch_id, {})) identity = { + "stable_game_id": running_snapshot.get("stable_game_id", ""), "game_path": running_snapshot.get("game_path", ""), "game_name": running_snapshot.get("game") or running_snapshot.get("game_name", ""), "steam_app_id": running_snapshot.get("steam_app_id", ""), @@ -477,12 +552,29 @@ def finish_session(launch_id, game_index, started, process): } with STATE_LOCK: state = load_state() - settings = state.get("settings", {}) + settings = copy.deepcopy(state.get("settings", {})) game = resolve_library_game(state, identity, fallback_index=game_index) or {} - game_path = str(game.get("path", "") or identity.get("game_path", "")) - original_game_name = str(game.get("name", "") or identity.get("game_name") or "Untitled") - exit_code = wait_for_exit(process, game, settings) + game_snapshot = copy.deepcopy(game) + original_game_name = str(game_snapshot.get("name", "") or identity.get("game_name") or "Untitled") + exit_code = wait_for_exit(process, game_snapshot, settings) seconds = max(1, int((datetime.now() - started).total_seconds())) + + if game_snapshot: + if settings.get("backup_on_close") and game_snapshot.get("save_paths"): + try: + backup_saves(game_snapshot, DATA.parent / "save-backups", label="on-close") + enforce_backup_limit(game_snapshot, DATA.parent / "save-backups", settings.get("save_backup_limit", 10)) + except (OSError, FileNotFoundError): + pass + try: + auto_attach_obs_recording(game_snapshot, started, settings) + except (OSError, ValueError, FileNotFoundError): + pass + try: + close_store_client(game_snapshot, settings) + except (OSError, ValueError): + pass + with STATE_LOCK: state = load_state() settings = state.get("settings", {}) @@ -490,18 +582,10 @@ def finish_session(launch_id, game_index, started, process): if game is not None: game["playtime_seconds"] = game.get("playtime_seconds", 0) + seconds apply_progress_automation(game, settings) - if settings.get("backup_on_close") and game.get("save_paths"): - try: - backup_saves(game, DATA.parent / "save-backups", label="on-close") - enforce_backup_limit(game, DATA.parent / "save-backups", settings.get("save_backup_limit", 10)) - except (OSError, FileNotFoundError): - pass - try: - auto_attach_obs_recording(game, started, settings) - except (OSError, ValueError, FileNotFoundError): - pass + for key in ("video_recording", "recording", "last_recording"): + if key in game_snapshot: + game[key] = game_snapshot[key] game_name = game.get("name", "Untitled") - close_store_client(game, settings) else: game_name = original_game_name session = { @@ -526,14 +610,9 @@ def finish_session(launch_id, game_index, started, process): pass if running.get("restart"): state = load_state() - index = next( - ( - index for index, game in enumerate(state["games"]) - if game.get("path") == game_path and game.get("name") == original_game_name - ), - None, - ) - if index is not None: + target = resolve_library_game(state, identity, fallback_index=game_index) + if target is not None: + index = state["games"].index(target) try: start_game(index) except (OSError, ValueError, IndexError): @@ -541,13 +620,14 @@ def finish_session(launch_id, game_index, started, process): def download_image(url, destination): - request = Request(url, headers={"User-Agent": "OpenBox/1"}) - with urlopen(request, timeout=15) as response: - if not response.headers.get_content_type().startswith("image/"): - raise ValueError("The media server did not return an image.") - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(response.read()) - return str(destination) + return str(download_file( + url, + destination, + expected_types=("image/",), + max_bytes=32 * 1024 * 1024, + timeout=15, + opener=urlopen, + )) def update_steam_metadata(game): @@ -590,36 +670,47 @@ def update_steam_metadata(game): def start_game(index): with STATE_LOCK: state = load_state() - game = state["games"][index] - args, cwd = build_launch(game, state["profiles"]) - if not os.environ.get("OPENBOX_SAFE_MODE"): - result = run_plugins(DATA.parent / "plugins", "before_launch", {"game": game, "args": args, "cwd": cwd}) - if not isinstance(result, dict): - raise ValueError("A plugin returned an invalid launch response.") - if result.get("cancel"): - raise ValueError(str(result.get("error") or "Launch canceled by a plugin.")) - args, cwd = result.get("args"), result.get("cwd") - if not isinstance(args, list) or not args or not all(isinstance(part, str) and part for part in args): - raise ValueError("A plugin returned an invalid launch command.") - if not isinstance(cwd, str) or not Path(cwd).is_dir(): - raise ValueError("A plugin returned an invalid working directory.") - process = subprocess.Popen(args, cwd=cwd, start_new_session=True) - started = datetime.now() - launch_id = secrets.token_urlsafe(8) - game["last_played"] = started.isoformat(timespec="seconds") - game["play_count"] = game.get("play_count", 0) + 1 - if not game.get("progress") and state.get("settings", {}).get("progress_on_first_play", "Playing"): - game["progress"] = state.get("settings", {}).get("progress_on_first_play", "Playing") + if index < 0 or index >= len(state["games"]): + raise IndexError("Game not found") + game = copy.deepcopy(state["games"][index]) + profiles = dict(state["profiles"]) + args, cwd = build_launch(game, profiles) + if not os.environ.get("OPENBOX_SAFE_MODE"): + result = run_plugins(DATA.parent / "plugins", "before_launch", {"game": game, "args": args, "cwd": cwd}) + if not isinstance(result, dict): + raise ValueError("A plugin returned an invalid launch response.") + if result.get("cancel"): + raise ValueError(str(result.get("error") or "Launch canceled by a plugin.")) + args, cwd = result.get("args"), result.get("cwd") + if not isinstance(args, list) or not args or not all(isinstance(part, str) and part for part in args): + raise ValueError("A plugin returned an invalid launch command.") + if not isinstance(cwd, str) or not Path(cwd).is_dir(): + raise ValueError("A plugin returned an invalid working directory.") + process = subprocess.Popen(args, cwd=cwd, start_new_session=True) + started = datetime.now() + launch_id = secrets.token_urlsafe(8) + stable_game_id = str(game.get("game_id") or "") + with STATE_LOCK: + state = load_state() + current = resolve_library_game(state, {"stable_game_id": stable_game_id}, fallback_index=index) + if current is None: + process.terminate() + raise IndexError("Game was removed while it was launching") + current["last_played"] = started.isoformat(timespec="seconds") + current["play_count"] = current.get("play_count", 0) + 1 + if not current.get("progress") and state.get("settings", {}).get("progress_on_first_play", "Playing"): + current["progress"] = state.get("settings", {}).get("progress_on_first_play", "Playing") save_state(state) entry = { "launch_id": launch_id, "game_id": index, - "game": game.get("name", "Untitled"), - "game_path": str(game.get("path", "")), - "steam_app_id": str(game.get("steam_app_id") or ""), - "heroic_app_id": str(game.get("heroic_app_id") or ""), - "lutris_id": str(game.get("lutris_id") or ""), - "gameyfin_id": str(game.get("gameyfin_id") or ""), + "stable_game_id": stable_game_id, + "game": current.get("name", "Untitled"), + "game_path": str(current.get("path", "")), + "steam_app_id": str(current.get("steam_app_id") or ""), + "heroic_app_id": str(current.get("heroic_app_id") or ""), + "lutris_id": str(current.get("lutris_id") or ""), + "gameyfin_id": str(current.get("gameyfin_id") or ""), "started": started.isoformat(timespec="seconds"), "pid": process.pid, "paused": False, @@ -686,6 +777,12 @@ def sync_cloud(): class Handler(BaseHTTPRequestHandler): server_version = "OpenBox/1" + MAX_BODY = 65536 + REQUEST_TIMEOUT = 30 + + def setup(self): + super().setup() + self.connection.settimeout(self.REQUEST_TIMEOUT) def log_message(self, *_): pass @@ -708,6 +805,49 @@ def send_bytes(self, status, data, content_type): self.end_headers() self.wfile.write(data) + def send_file(self, status, path, content_type=None): + path = Path(path) + size = path.stat().st_size + start, end = 0, size - 1 + response_status = status + range_header = self.headers.get("Range", "") + if range_header.startswith("bytes="): + spec = range_header[6:].split(",", 1)[0].strip() + if "-" not in spec: + raise ValueError("Invalid byte range.") + left, right = spec.split("-", 1) + if left: + start = int(left) + end = int(right) if right else end + elif right: + length = int(right) + start = max(0, size - length) + if start < 0 or start >= size or end < start: + self.send_response(416) + self.headers_common(content_type or "application/octet-stream") + self.send_header("Content-Range", f"bytes */{size}") + self.end_headers() + return + end = min(end, size - 1) + response_status = 206 + length = max(0, end - start + 1) + self.send_response(response_status) + self.headers_common(content_type or mimetypes.guess_type(path.name)[0] or "application/octet-stream") + self.send_header("Accept-Ranges", "bytes") + self.send_header("Content-Length", str(length)) + if response_status == 206: + self.send_header("Content-Range", f"bytes {start}-{end}/{size}") + self.end_headers() + with path.open("rb") as source: + source.seek(start) + remaining = length + while remaining: + chunk = source.read(min(1024 * 1024, remaining)) + if not chunk: + break + self.wfile.write(chunk) + remaining -= len(chunk) + def send_json(self, status, payload): self.send_bytes(status, json.dumps(payload).encode(), "application/json; charset=utf-8") @@ -717,10 +857,19 @@ def authorized(self): return secrets.compare_digest(provided, TOKEN) def body(self): - length = int(self.headers.get("Content-Length", "0")) - if length > 65536: + raw_length = self.headers.get("Content-Length", "0") + try: + length = int(raw_length) + except (TypeError, ValueError) as error: + raise ValueError("Content-Length must be a valid number.") from error + if length < 0: + raise ValueError("Content-Length must not be negative.") + if length > self.MAX_BODY: raise ValueError("Request is too large.") - return json.loads(self.rfile.read(length) or b"{}") + raw = self.rfile.read(length) + if len(raw) != length: + raise ValueError("Request body was truncated.") + return json.loads(raw or b"{}") def _do_GET(self): parsed = urlparse(self.path) @@ -778,9 +927,10 @@ def _do_GET(self): self.send_json(403, {"error": "Unauthorized"}) return try: - index = int(parse_qs(parsed.query)["id"][0]) - games = load_state()["games"] - related = related_game_ids(games, index) + query = parse_qs(parsed.query) + state = load_state() + index = state["games"].index(game_from_query(state, query)) + related = related_game_ids(state["games"], index) self.send_json(200, {"ids": related}) except (KeyError, IndexError, ValueError): self.send_json(404, {"error": "Game not found"}) @@ -801,7 +951,8 @@ def _do_GET(self): self.send_json(403, {"error": "Unauthorized"}) return try: - game = load_state()["games"][int(parse_qs(parsed.query)["id"][0])] + query = parse_qs(parsed.query) + game = game_from_query(load_state(), query) backups = [{"name": path.name, "size": path.stat().st_size} for path in list_backups(game, DATA.parent / "save-backups")] self.send_json(200, {"backups": backups}) except (KeyError, IndexError, ValueError): @@ -812,7 +963,8 @@ def _do_GET(self): self.send_json(403, {"error": "Unauthorized"}) return try: - game = load_state()["games"][int(parse_qs(parsed.query)["id"][0])] + query = parse_qs(parsed.query) + game = game_from_query(load_state(), query) configured = set(game.get("save_paths", [])) candidates = [ item for item in discover_save_paths(game) + extra_save_candidates(game) @@ -907,7 +1059,7 @@ def _do_GET(self): return try: query = parse_qs(parsed.query) - game = load_state()["games"][int(query["id"][0])] + game = game_from_query(load_state(), query) title = query.get("q", [game.get("name", "")])[0] results = search_games(METADATA_DATABASE, title, game.get("platform", "")) self.send_json(200, {"results":results}) @@ -954,7 +1106,7 @@ def _do_GET(self): try: if not badge.is_file(): download_image(f"https://media.retroachievements.org/Badge/{badge.name}", badge) - self.send_bytes(200, badge.read_bytes(), "image/png") + self.send_file(200, badge, "image/png") except (OSError, ValueError): self.send_json(404, {"error": "Badge not found"}) return @@ -964,7 +1116,7 @@ def _do_GET(self): return query = parse_qs(parsed.query) try: - game = load_state()["games"][int(query["id"][0])] + game = game_from_query(load_state(), query) kind = query["kind"][0] if kind == "screenshot": index = int(query["index"][0]) @@ -979,7 +1131,7 @@ def _do_GET(self): raise ValueError if not media.is_file(): raise FileNotFoundError - self.send_bytes(200, media.read_bytes(), mimetypes.guess_type(media.name)[0] or "application/octet-stream") + self.send_file(200, media) except (KeyError, IndexError, ValueError, FileNotFoundError): self.send_json(404, {"error": "Media not found"}) return @@ -989,7 +1141,7 @@ def _do_GET(self): return query = parse_qs(parsed.query) try: - game = load_state()["games"][int(query["id"][0])] + game = game_from_query(load_state(), query) document = game.get("documents", [])[int(query["index"][0])] path = Path(document["path"]) if not path.is_file(): @@ -1071,7 +1223,7 @@ def _do_GET(self): self.send_json(403, {"error": "Unauthorized"}) return try: - game = load_state()["games"][int(parse_qs(parsed.query)["id"][0])] + game = game_from_query(load_state(), parse_qs(parsed.query)) self.send_json(200, {"scores": read_local_highscores(game)}) except (KeyError, IndexError, ValueError): self.send_json(404, {"error": "Game not found"}) @@ -1304,6 +1456,8 @@ def _do_POST(self): self.save_profiles(payload) elif route == "/api/settings": self.save_settings(payload) + elif route == "/api/state/recover": + self.recover_state() elif route == "/api/image-group": self.save_image_group(payload) elif route == "/api/cloud/sync": @@ -1428,6 +1582,9 @@ def _handle_request(self, method): LOGGER.debug("HTTP %s %s started", method, path) try: getattr(self, f"_{method}")() + except StateCorruptError as error: + LOGGER.error("OpenBox state is unavailable: %s", error) + self.send_json(503, {"error": "OpenBox library data needs recovery before this operation can continue."}) except Exception: LOGGER.exception("Unhandled HTTP %s %s", method, path) self.send_json(500, {"error": "Unexpected server error. Copy the diagnostic log from Settings and include it in your report."}) @@ -1439,7 +1596,14 @@ def do_POST(self): self._handle_request("do_POST") def launch(self, payload): - self.send_json(200, {"ok": True, **start_game(int(payload["id"]))}) + if payload.get("id") is None and not payload.get("game_id"): + raise ValueError("Game id is required.") + legacy_id = int(payload["id"]) if payload.get("id") is not None else int(payload.get("legacy_id", 0)) + if payload.get("game_id"): + state = load_state() + game = game_from_payload(state, payload) + legacy_id = state["games"].index(game) + self.send_json(200, {"ok": True, **start_game(legacy_id)}) def control_session(self, payload): launch_id = str(payload.get("launch_id", "")) @@ -1449,7 +1613,7 @@ def control_session(self, payload): def favorite(self, payload): with STATE_LOCK: state = load_state() - game = state["games"][int(payload["id"])] + game = game_from_payload(state, payload) game["favorite"] = not game.get("favorite", False) save_state(state) self.send_json(200, {"favorite": game["favorite"]}) @@ -1503,11 +1667,12 @@ def save_game(self, payload): raise ValueError("Path must point to an existing local file.") with STATE_LOCK: state = load_state() - if payload.get("id") is None: + if payload.get("id") is None and not payload.get("game_id"): game["added_at"] = datetime.now().isoformat(timespec="seconds") state["games"].append(game) else: - existing = state["games"][int(payload["id"])] + existing = game_from_payload(state, payload) + game["game_id"] = existing.get("game_id", game.get("game_id", "")) existing.update(game) save_state(state) self.send_json(200, {"ok": True}) @@ -1523,14 +1688,13 @@ def delete_game(self, payload): delete_media = bool(payload.get("delete_media")) with STATE_LOCK: state = load_state() - game = state["games"].pop(int(payload["id"])) + game = game_from_payload(state, payload) + state["games"].remove(game) if delete_media: for path in game_media_paths(game): try: - target = Path(path).expanduser() - if target.is_file(): - target.unlink() - except OSError: + remove_file_if_safe(Path(path), DATA.parent) + except (OSError, ValueError): pass save_state(state) self.send_json(200, {"removed": game.get("name", "")}) @@ -1673,10 +1837,13 @@ def import_arcade_games(self, payload): self.send_json(200, {"added": len(new_games), "found": len(imported), "sets": counts}) def steam_metadata(self, payload): + state = load_state() + target = copy.deepcopy(game_from_payload(state, payload)) + update_steam_metadata(target) with STATE_LOCK: state = load_state() - game = state["games"][int(payload["id"])] - update_steam_metadata(game) + game = game_from_payload(state, {"game_id": target.get("game_id"), **payload}) + game.update(target) save_state(state) self.send_json(200, {"ok": True}) @@ -1698,18 +1865,19 @@ def worker(): METADATA_JOB.clear() METADATA_JOB.update(job) - threading.Thread(target=worker, daemon=True).start() + JOB_MANAGER.submit("metadata", worker) self.send_json(202, {"state":"downloading"}) def apply_metadata(self, payload): if not METADATA_DATABASE.is_file(): raise ValueError("Download the metadata database first.") - index = int(payload["id"]) media_types = payload.get("media", []) if not isinstance(media_types, list) or not set(media_types) <= {"cover", "background", "screenshots"}: raise ValueError("Invalid media selection.") state = load_state() - original = dict(state["games"][index]) + original_game = game_from_payload(state, payload) + stable_game_id = original_game.get("game_id") + original = dict(original_game) updated = apply_game_metadata( dict(original), METADATA_DATABASE, int(payload["database_id"]), media_types, DATA.parent / "media/launchbox", bool(payload.get("overwrite")), @@ -1718,7 +1886,7 @@ def apply_metadata(self, payload): changes = {key:value for key,value in updated.items() if original.get(key) != value} with STATE_LOCK: state = load_state() - state["games"][index].update(changes) + game_from_payload(state, {"game_id": stable_game_id}).update(changes) save_state(state) self.send_json(200, {"updated":sorted(changes)}) @@ -1740,36 +1908,37 @@ def bulk_media(self, payload): def worker(): state = load_state() targets = [ - index for index, game in enumerate(state["games"]) + (str(game.get("game_id")), str(game.get("launchbox_db_id"))) + for game in state["games"] if game.get("launchbox_db_id") and (platform == "all" or game.get("platform") == platform) ] with PROCESS_LOCK: MEDIA_JOB["total"] = len(targets) updated_count, errors = 0, [] - for current, index in enumerate(targets, 1): + for current, (stable_id, database_id) in enumerate(targets, 1): original = {} try: state = load_state() - original = dict(state["games"][index]) + original = dict(game_from_payload(state, {"game_id": stable_id})) updated = apply_game_metadata( - dict(original), METADATA_DATABASE, int(original["launchbox_db_id"]), media_types, + dict(original), METADATA_DATABASE, int(database_id), media_types, DATA.parent / "media/launchbox", overwrite, ) changes = {key:value for key,value in updated.items() if original.get(key) != value} if changes: with STATE_LOCK: state = load_state() - state["games"][index].update(changes) + game_from_payload(state, {"game_id": stable_id}).update(changes) save_state(state) updated_count += 1 except (OSError, ValueError, sqlite3.Error) as error: - errors.append(f"{original.get('name', index)}: {error}") + errors.append(f"{original.get('name', stable_id)}: {error}") with PROCESS_LOCK: MEDIA_JOB.update({"current":current, "updated":updated_count, "errors":errors[-20:]}) with PROCESS_LOCK: MEDIA_JOB["state"] = "done" - threading.Thread(target=worker, daemon=True).start() + JOB_MANAGER.submit("media-bulk", worker) self.send_json(202, {"state":"running"}) def save_profiles(self, payload): @@ -1787,6 +1956,10 @@ def save_profiles(self, payload): save_state(state) self.send_json(200, {"saved": len(clean)}) + def recover_state(self): + state = recover_library_state() + self.send_json(200, {"ok": True, "games": len(state.get("games", []))}) + def save_settings(self, payload): with STATE_LOCK: existing_settings = dict(load_state().get("settings", {})) @@ -1903,11 +2076,8 @@ def save_settings(self, payload): progress_on_first_play = str(merged.get("progress_on_first_play", "Playing")).strip() if progress_on_first_play and progress_on_first_play not in PROGRESS: raise ValueError("Unknown progress value for first play.") - with STATE_LOCK: - state = load_state() - settings = state.setdefault("settings", {}) - gameyfin_password = str(merged.get("gameyfin_password", "")).strip() - settings.update({ + gameyfin_password = str(merged.get("gameyfin_password", "")).strip() + normalized_settings = { "watch_folders": clean_folders, "screensaver_seconds": seconds, "controller_map": clean_mapping, @@ -1956,7 +2126,17 @@ def save_settings(self, payload): "tracking_frequency": tracking_frequency, "progress_on_first_play": progress_on_first_play, "auto_close_store_clients": bool(merged.get("auto_close_store_clients", False)), - }) + } + with STATE_LOCK: + state = load_state() + settings = state.setdefault("settings", {}) + incoming_keys = { + key for key, value in payload.items() + if key != "gameyfin_password" or str(value).strip() + } + for key, value in normalized_settings.items(): + if key in incoming_keys or key not in settings: + settings[key] = value save_state(state) self.send_json(200, public_settings(state)) @@ -2056,14 +2236,15 @@ def ra_game(self, payload): credentials = load_ra_credentials(DATA.parent) if not credentials: raise ValueError("Configure RetroAchievements first.") - index = int(payload["id"]) state = load_state() - game = state["games"][index] + game = copy.deepcopy(game_from_payload(state, payload)) + stable_game_id = game.get("game_id") game_id, digest = match_ra_game(game, credentials, DATA.parent / "cache/retroachievements") with STATE_LOCK: state = load_state() - state["games"][index]["ra_game_id"] = str(game_id) - state["games"][index]["ra_hash"] = digest + target = game_from_payload(state, {"game_id": stable_game_id}) + target["ra_game_id"] = str(game_id) + target["ra_hash"] = digest save_state(state) progress = ra_game_progress(game_id, credentials) progress["game_id"] = game_id @@ -2087,20 +2268,24 @@ def apply_media_pack_route(self, payload): self.send_json(200, {"pack": pack, "settings": public_settings(state)}) def download_trailer(self, payload): - index = int(payload["id"]) + state = load_state() + target = copy.deepcopy(game_from_payload(state, payload)) + path = download_steam_trailer(target, DATA.parent / "media") with STATE_LOCK: state = load_state() - game = state["games"][index] - path = download_steam_trailer(game, DATA.parent / "media") + game = game_from_payload(state, {"game_id": target.get("game_id")}) + game.update(target) save_state(state) self.send_json(200, {"video_trailer": path}) def download_gog_route(self, payload): - index = int(payload["id"]) + state = load_state() + target = copy.deepcopy(game_from_payload(state, payload)) + download_gog_media(target, DATA.parent / "media") with STATE_LOCK: state = load_state() - game = state["games"][index] - download_gog_media(game, DATA.parent / "media") + game = game_from_payload(state, {"game_id": target.get("game_id")}) + game.update(target) save_state(state) self.send_json(200, {"cover": game.get("cover", ""), "background": game.get("background", "")}) @@ -2134,7 +2319,7 @@ def remove_plugin(self, payload): def launch_extra(self, payload): state = load_state() - game = state["games"][int(payload["id"])] + game = game_from_payload(state, payload) kind = payload.get("kind") if kind not in {"applications", "versions", "documents"}: raise ValueError("Unknown extra type.") @@ -2155,13 +2340,13 @@ def launch_extra(self, payload): self.send_json(200, {"ok": True}) def backup_game_saves(self, payload): - game = load_state()["games"][int(payload["id"])] + game = game_from_payload(load_state(), payload) archive = backup_saves(game, DATA.parent / "save-backups") removed = enforce_backup_limit(game, DATA.parent / "save-backups", load_state().get("settings", {}).get("save_backup_limit", 10)) self.send_json(200, {"backup": archive.name, "trimmed": removed}) def restore_game_saves(self, payload): - game = load_state()["games"][int(payload["id"])] + game = game_from_payload(load_state(), payload) archive = restore_saves(game, DATA.parent / "save-backups", str(payload["backup"])) self.send_json(200, {"restored": archive.name}) @@ -2171,7 +2356,7 @@ def add_game_save_path(self, payload): raise FileNotFoundError("Save path does not exist.") with STATE_LOCK: state = load_state() - paths = state["games"][int(payload["id"])].setdefault("save_paths", []) + paths = game_from_payload(state, payload).setdefault("save_paths", []) if str(path) not in paths: paths.append(str(path)) save_state(state) @@ -2446,13 +2631,15 @@ def save_emumovies(self, payload): def emumovies_download(self, payload): credentials = load_emumovies_credentials(DATA.parent) - index = int(payload["id"]) + state = load_state() + target = copy.deepcopy(game_from_payload(state, payload)) + path = download_emumovies_media( + target, credentials, DATA.parent / "media", str(payload.get("type", "box")), + ) with STATE_LOCK: state = load_state() - game = state["games"][index] - path = download_emumovies_media( - game, credentials, DATA.parent / "media", str(payload.get("type", "box")), - ) + game = game_from_payload(state, {"game_id": target.get("game_id")}) + game.update(target) game["cover"] = path save_state(state) self.send_json(200, {"path": path}) @@ -2460,41 +2647,52 @@ def emumovies_download(self, payload): def cleanup_media(self, payload): groups = find_duplicate_media(load_state()["games"]) apply = bool(payload.get("apply")) - deleted = cleanup_duplicates(groups, dry_run=not apply) + deleted = cleanup_duplicates(groups, dry_run=not apply, allowed_roots=[DATA.parent]) self.send_json(200, {"groups": len(groups), "paths": deleted, "applied": apply}) def take_screenshot(self, payload): - index = int(payload["id"]) state = load_state() - game = state["games"][index] + game = game_from_payload(state, payload) + stable_game_id = game.get("game_id") destination = DATA.parent / "media" / "captures" / f"{Path(game.get('path', 'game')).stem}-{datetime.now().strftime('%Y%m%d-%H%M%S')}.png" path = capture_screenshot(destination) with STATE_LOCK: state = load_state() - screenshots = state["games"][index].setdefault("screenshots", []) + screenshots = game_from_payload(state, {"game_id": stable_game_id}).setdefault("screenshots", []) if path not in screenshots: screenshots.append(path) save_state(state) self.send_json(200, {"path": path}) def obs_attach(self, payload): - index = int(payload["id"]) video_path = str(payload.get("path", "")).strip() + state = load_state() + target = copy.deepcopy(game_from_payload(state, payload)) + path = attach_recording(target, video_path) with STATE_LOCK: state = load_state() - game = state["games"][index] - path = attach_recording(game, video_path) + game = game_from_payload(state, {"game_id": target.get("game_id")}) + game.update(target) save_state(state) self.send_json(200, {"path": path, "obs": obs_recording_status()}) def apply_save_scan(self, payload): state = load_state() found = scan_all_saves(state["games"]) + found_by_id = { + str(state["games"][index].get("game_id")): paths + for index, paths in found.items() + if 0 <= index < len(state["games"]) + } updated = 0 with STATE_LOCK: state = load_state() - for index, paths in found.items(): - save_paths = state["games"][index].setdefault("save_paths", []) + for stable_id, paths in found_by_id.items(): + try: + game = game_from_payload(state, {"game_id": stable_id}) + except IndexError: + continue + save_paths = game.setdefault("save_paths", []) for path in paths: if path not in save_paths: save_paths.append(path) @@ -2550,6 +2748,13 @@ def install_gameyfin(self, payload): if not game_id: raise ValueError("gameyfin_id is required.") library_id = payload.get("library_id") + stable_library_id = "" + if library_id is not None: + try: + library_state = load_state() + stable_library_id = str(game_from_payload(library_state, {"id": library_id}).get("game_id") or "") + except (ValueError, IndexError): + stable_library_id = str(library_id) job_key = f"gameyfin:{game_id}" with PROCESS_LOCK: job = INSTALLS.get(job_key, {}) @@ -2571,6 +2776,8 @@ def worker(): if str(game.get("gameyfin_id") or "") == game_id: target = game break + if target is None and stable_library_id: + target = resolve_library_game(state, {"stable_game_id": stable_library_id}) if target is None and library_id is not None: try: index = int(library_id) @@ -2596,13 +2803,16 @@ def worker(): self.send_json(202, {"state": "installing", "gameyfin_id": game_id}) def uninstall_gameyfin(self, payload): - index = int(payload["id"]) + state = load_state() + original = game_from_payload(state, payload) + target = copy.deepcopy(original) + if not target.get("gameyfin_id"): + raise ValueError("This game is not a Gameyfin entry.") + result = uninstall_gameyfin_game(target) with STATE_LOCK: state = load_state() - game = state["games"][index] - if not game.get("gameyfin_id"): - raise ValueError("This game is not a Gameyfin entry.") - result = uninstall_gameyfin_game(game) + game = game_from_payload(state, {"game_id": target.get("game_id")}) + game.update(target) save_state(state) self.send_json(200, result) @@ -2610,7 +2820,7 @@ def run_ludusavi_tool(self, payload): settings = load_state().get("settings", {}) game_name = str(payload.get("name", "")) if "id" in payload and not game_name: - game_name = load_state()["games"][int(payload["id"])].get("name", "") + game_name = game_from_payload(load_state(), payload).get("name", "") result = run_ludusavi( str(payload.get("action", "backup")), game_name=game_name, @@ -2621,23 +2831,21 @@ def run_ludusavi_tool(self, payload): def run_hoard_tool(self, payload): game_name = str(payload.get("name", "")) if "id" in payload and not game_name: - game_name = load_state()["games"][int(payload["id"])].get("name", "") + game_name = game_from_payload(load_state(), payload).get("name", "") result = run_hoard(str(payload.get("action", "backup")), game_name=game_name) self.send_json(200, result) def export_game_highscores(self, payload): - index = int(payload["id"]) state = load_state() - game = state["games"][index] + game = game_from_payload(state, payload) export_dir = DATA.parent / "highscores" / re.sub(r"[^a-z0-9]+", "-", str(game.get("name", "game")).casefold()).strip("-") result = export_highscores(game, export_dir) self.send_json(200, result) def import_game_highscores(self, payload): - index = int(payload["id"]) import_dir = str(payload.get("path", "")).strip() state = load_state() - game = state["games"][index] + game = game_from_payload(state, payload) restored = import_highscores(game, import_dir) self.send_json(200, {"restored": restored}) @@ -2702,8 +2910,8 @@ def main(): server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) run_configured_commands("startup_commands") port = server.server_address[1] - (DATA.parent / "server.port").write_text(str(port)) - (DATA.parent / "server.token").write_text(TOKEN) + secure_text_write(DATA.parent / "server.port", str(port)) + secure_text_write(DATA.parent / "server.token", TOKEN) url = f"http://127.0.0.1:{port}/?token={TOKEN}" force_game_mode = "--game-mode" in sys.argv guest = is_gamescope_guest(force=force_game_mode) @@ -2730,6 +2938,8 @@ def main(): finally: WATCH_STOP.set() server.server_close() + (DATA.parent / "server.token").unlink(missing_ok=True) + (DATA.parent / "server.port").unlink(missing_ok=True) run_configured_commands("shutdown_commands")