diff --git a/docs/config.md b/docs/config.md index 358a65cad..b29400ea4 100644 --- a/docs/config.md +++ b/docs/config.md @@ -46,7 +46,7 @@ logger.info("This will be shown") # (2)! | `cacheToDisk` | `bool` | `True` | Enable disk-based caching for file uploads | | `cacheTimeout` | `float` | `1` | Cache operation timeout in seconds | | `cacheLocation` | `Path` | `~/.cache/rapidata/upload_cache` | Directory for cache storage (immutable) | -| `cacheShards` | `int` | `128` | Number of cache shards for parallel access (immutable) | +| `cacheShards` | `int` | `32` | Number of disk-cache shards for concurrent access (immutable). Each shard holds open file handles — see [Too many open files](#too-many-open-files) | | `batchSize` | `int` | `1000` | Number of URLs per batch (100–5000) | | `batchPollInterval` | `float` | `0.5` | Batch polling interval in seconds | | `compression` | `CompressionConfig \| None` | `None` | Per-upload image-compression settings; see [Compression override](#compression-override) below. | @@ -67,6 +67,24 @@ rapidata_config.upload.compression = CompressionConfig( Any field left as `None` falls back to the server-side default. Currently applies to single-asset uploads (`/asset/file` and `/asset/url`); batched URL uploads will pick the override up in a follow-up after the OpenAPI client regenerates. +#### Too many open files + +Uploading local files opens file descriptors — for the on-disk upload cache (one set of handles per `cacheShards`), the worker pool (`maxWorkers`), and the HTTP connections. On systems with a low `ulimit -n` (1024 is common), a large or highly concurrent upload can exhaust the limit and fail with `OSError: [Errno 24] Too many open files`. + +Two ways to resolve it: + +- **Raise the OS limit** (per shell): `ulimit -n 8192`. +- **Lower the SDK's footprint** — reduce the cache shards and/or the worker pool: + + ```bash + export RAPIDATA_cacheShards=16 # fewer cache shards = fewer open handles + export RAPIDATA_maxWorkers=10 # fewer concurrent uploads + ``` + + `cacheShards` is immutable at runtime, so set it via the environment variable (or a `.env` file); `maxWorkers` can also be set in code (`rapidata_config.upload.maxWorkers = 10`). + +The default `cacheShards` of 32 keeps a single upload well under a 1024 limit; lower it further only if you run many upload processes concurrently against the same limit. + ## Environment Variables Every configuration field can also be set through an environment variable prefixed with `RAPIDATA_` followed by the field name (e.g. `RAPIDATA_maxWorkers`). This is useful for CI/CD pipelines, containers, or any context where you want to configure the SDK without changing code. @@ -107,7 +125,7 @@ RAPIDATA_maxRetries=3 RAPIDATA_cacheToDisk=true RAPIDATA_cacheTimeout=1 RAPIDATA_cacheLocation=~/.cache/rapidata/upload_cache -RAPIDATA_cacheShards=128 +RAPIDATA_cacheShards=32 RAPIDATA_batchSize=1000 RAPIDATA_batchPollInterval=0.5 diff --git a/docs/error_handling.md b/docs/error_handling.md index eac529746..2b5df1027 100644 --- a/docs/error_handling.md +++ b/docs/error_handling.md @@ -174,3 +174,9 @@ except FailedUploadException as e: ``` Note that this only re-uploads the datapoints; it does not create the job definition. Prefer `exception.retry()` unless you specifically need the lower-level control. + +## Too many open files + +If uploads fail with `OSError: [Errno 24] Too many open files`, the process has hit its file-descriptor limit — the upload cache, worker pool, and HTTP connections all consume descriptors. When the SDK detects this it appends a hint to the `FailedUploadException` message pointing at the relevant knobs. + +Raise the OS limit (`ulimit -n 8192`) or lower the SDK's footprint with `RAPIDATA_cacheShards` / `RAPIDATA_maxWorkers`. See [Too many open files](config.md#too-many-open-files) in the configuration guide for details. diff --git a/src/rapidata/rapidata_client/config/upload_config.py b/src/rapidata/rapidata_client/config/upload_config.py index ec3849a37..c4d0bcfa0 100644 --- a/src/rapidata/rapidata_client/config/upload_config.py +++ b/src/rapidata/rapidata_client/config/upload_config.py @@ -91,9 +91,12 @@ class UploadConfig(BaseModel): cacheTimeout (float): Cache operation timeout in seconds. Defaults to 0.1. cacheLocation (Path): Directory for cache storage. Defaults to ~/.cache/rapidata/upload_cache. This is immutable. Only used for file uploads when cacheToDisk=True. - cacheShards (int): Number of cache shards for parallel access. Defaults to 128. - Higher values improve concurrency but increase file handles. Must be positive. - This is immutable. Only used for file uploads when cacheToDisk=True. + cacheShards (int): Number of disk-cache shards for concurrent file-cache access. Defaults to 32. + Each shard is a separate on-disk store that holds open file handles, so a higher value + raises the process's file-descriptor count — which can exceed a low ``ulimit -n`` and + surface as "Too many open files". 32 comfortably covers the default ``maxWorkers`` of 25. + Must be positive. Immutable at runtime — set it via the ``RAPIDATA_cacheShards`` environment + variable. Only used for file uploads when cacheToDisk=True. enableBatchUpload (bool): Enable batch URL uploading (two-step process). Defaults to True. batchSize (int): Number of URLs per batch (100-5000). Defaults to 1000. batchPollInterval (float): Polling interval in seconds. Defaults to 0.5. @@ -134,8 +137,9 @@ def _apply_env_vars(cls, data: Any) -> Any: frozen=True, ) cacheShards: int = Field( - default=128, + default=32, frozen=True, + description="Disk-cache shards. Each opens file handles; keep low enough for ulimit -n.", ) batchSize: int = Field( default=1000, diff --git a/src/rapidata/rapidata_client/exceptions/failed_upload_exception.py b/src/rapidata/rapidata_client/exceptions/failed_upload_exception.py index 566e96a86..1a1e8a0eb 100644 --- a/src/rapidata/rapidata_client/exceptions/failed_upload_exception.py +++ b/src/rapidata/rapidata_client/exceptions/failed_upload_exception.py @@ -1,4 +1,5 @@ from __future__ import annotations +import errno from rapidata.rapidata_client.datapoints._datapoint import Datapoint from .failed_upload import FailedUpload from typing import TYPE_CHECKING, Optional @@ -95,6 +96,20 @@ def failures_by_reason(self) -> dict[str, list[Datapoint]]: grouped[failed_upload.error_message].append(failed_upload.item) return dict(grouped) + def _has_fd_exhaustion_failure(self) -> bool: + """Whether any failure looks like the OS running out of file descriptors. + + Checks the errno first (EMFILE), falling back to the message text for + cases where the OSError was wrapped or stringified before it reached us. + """ + for fu in self._failed_uploads: + exc = fu.exception + if isinstance(exc, OSError) and exc.errno == errno.EMFILE: + return True + if "too many open files" in fu.error_message.lower(): + return True + return False + def __str__(self) -> str: total = len(self._failed_uploads) if total == 0: @@ -119,6 +134,16 @@ def __str__(self) -> str: lines.append(" ]") failed_upload_message = "\n".join(lines) + if self._has_fd_exhaustion_failure(): + failed_upload_message += ( + "\n\nThese failures look like file-descriptor exhaustion ('Too many open " + "files'). The upload cache and worker pool each hold open file handles, so a " + "large or concurrent upload can exceed a low 'ulimit -n'. To fix, either raise " + "the OS limit (e.g. 'ulimit -n 8192') or lower the SDK's footprint via the " + "'RAPIDATA_cacheShards' (default 32) and 'RAPIDATA_maxWorkers' (default 25) " + "environment variables. See " + "https://docs.rapidata.ai/3.x/config/#upload-configuration-options" + ) if self.machine is not None: failed_upload_message += ( "\n\nNo job definition was created because the upload stayed outside the " diff --git a/tests/rapidata_client/exceptions/test_failed_upload_exception.py b/tests/rapidata_client/exceptions/test_failed_upload_exception.py new file mode 100644 index 000000000..0a39fb223 --- /dev/null +++ b/tests/rapidata_client/exceptions/test_failed_upload_exception.py @@ -0,0 +1,51 @@ +"""Tests for FailedUploadException message rendering. + +Focused on the file-descriptor-exhaustion hint: when an upload fails because the +process ran out of file descriptors, the exception message must point the user +at the knobs (cacheShards / maxWorkers / ulimit) rather than only listing the +raw OSError. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from rapidata.rapidata_client.exceptions.failed_upload import FailedUpload +from rapidata.rapidata_client.exceptions.failed_upload_exception import ( + FailedUploadException, +) + + +def _exception(failed_uploads: list[FailedUpload]) -> FailedUploadException: + return FailedUploadException(dataset=MagicMock(), failed_uploads=failed_uploads) + + +def test_hint_added_when_errno_is_emfile() -> None: + exc = OSError(24, "Too many open files") + failure = FailedUpload.from_exception("img.jpg", exc) + + message = str(_exception([failure])) + + assert "file-descriptor exhaustion" in message + assert "RAPIDATA_cacheShards" in message + assert "RAPIDATA_maxWorkers" in message + assert "ulimit -n" in message + + +def test_hint_added_when_only_message_matches() -> None: + # Some layers stringify the OSError before it reaches us, losing the errno. + failure = FailedUpload( + item="img.jpg", + error_message="[Errno 24] Too many open files: 'img.jpg'", + error_type="OSError", + ) + + assert "file-descriptor exhaustion" in str(_exception([failure])) + + +def test_no_hint_for_unrelated_failures() -> None: + failure = FailedUpload( + item="img.jpg", error_message="404 not found", error_type="RapidataError" + ) + + assert "file-descriptor exhaustion" not in str(_exception([failure]))