fix(upload): reduce disk-cache file-descriptor use to avoid EMFILE

docs/config.md

@@ -46,7 +46,7 @@ logger.info("This will be shown") # (2)!
 | `cacheToDisk` | `bool` | `True` | Enable disk-based caching for file uploads |
 | `cacheTimeout` | `float` | `1` | Cache operation timeout in seconds |
 | `cacheLocation` | `Path` | `~/.cache/rapidata/upload_cache` | Directory for cache storage (immutable) |
-| `cacheShards` | `int` | `128` | Number of cache shards for parallel access (immutable) |
+| `cacheShards` | `int` | `32` | Number of disk-cache shards for concurrent access (immutable). Each shard holds open file handles — see [Too many open files](#too-many-open-files) |
 | `batchSize` | `int` | `1000` | Number of URLs per batch (100–5000) |
 | `batchPollInterval` | `float` | `0.5` | Batch polling interval in seconds |
 | `compression` | `CompressionConfig \| None` | `None` | Per-upload image-compression settings; see [Compression override](#compression-override) below. |
@@ -67,6 +67,24 @@ rapidata_config.upload.compression = CompressionConfig(
 
 Any field left as `None` falls back to the server-side default. Currently applies to single-asset uploads (`/asset/file` and `/asset/url`); batched URL uploads will pick the override up in a follow-up after the OpenAPI client regenerates.
 
+#### Too many open files
+
+Uploading local files opens file descriptors — for the on-disk upload cache (one set of handles per `cacheShards`), the worker pool (`maxWorkers`), and the HTTP connections. On systems with a low `ulimit -n` (1024 is common), a large or highly concurrent upload can exhaust the limit and fail with `OSError: [Errno 24] Too many open files`.
+
+Two ways to resolve it:
+
+- **Raise the OS limit** (per shell): `ulimit -n 8192`.
+- **Lower the SDK's footprint** — reduce the cache shards and/or the worker pool:
+
+    ```bash
+    export RAPIDATA_cacheShards=16   # fewer cache shards = fewer open handles
+    export RAPIDATA_maxWorkers=10    # fewer concurrent uploads
+    ```
+
+    `cacheShards` is immutable at runtime, so set it via the environment variable (or a `.env` file); `maxWorkers` can also be set in code (`rapidata_config.upload.maxWorkers = 10`).
+
+The default `cacheShards` of 32 keeps a single upload well under a 1024 limit; lower it further only if you run many upload processes concurrently against the same limit.
+
 ## Environment Variables
 
 Every configuration field can also be set through an environment variable prefixed with `RAPIDATA_` followed by the field name (e.g. `RAPIDATA_maxWorkers`). This is useful for CI/CD pipelines, containers, or any context where you want to configure the SDK without changing code.
@@ -107,7 +125,7 @@ RAPIDATA_maxRetries=3
 RAPIDATA_cacheToDisk=true
 RAPIDATA_cacheTimeout=1
 RAPIDATA_cacheLocation=~/.cache/rapidata/upload_cache
-RAPIDATA_cacheShards=128
+RAPIDATA_cacheShards=32
 RAPIDATA_batchSize=1000
 RAPIDATA_batchPollInterval=0.5
 

docs/error_handling.md

@@ -174,3 +174,9 @@ except FailedUploadException as e:
 ```
 
 Note that this only re-uploads the datapoints; it does not create the job definition. Prefer `exception.retry()` unless you specifically need the lower-level control.
+
+## Too many open files
+
+If uploads fail with `OSError: [Errno 24] Too many open files`, the process has hit its file-descriptor limit — the upload cache, worker pool, and HTTP connections all consume descriptors. When the SDK detects this it appends a hint to the `FailedUploadException` message pointing at the relevant knobs.
+
+Raise the OS limit (`ulimit -n 8192`) or lower the SDK's footprint with `RAPIDATA_cacheShards` / `RAPIDATA_maxWorkers`. See [Too many open files](config.md#too-many-open-files) in the configuration guide for details.

src/rapidata/rapidata_client/config/upload_config.py

@@ -91,9 +91,12 @@ class UploadConfig(BaseModel):
         cacheTimeout (float): Cache operation timeout in seconds. Defaults to 0.1.
         cacheLocation (Path): Directory for cache storage. Defaults to ~/.cache/rapidata/upload_cache.
             This is immutable. Only used for file uploads when cacheToDisk=True.
-        cacheShards (int): Number of cache shards for parallel access. Defaults to 128.
-            Higher values improve concurrency but increase file handles. Must be positive.
-            This is immutable. Only used for file uploads when cacheToDisk=True.
+        cacheShards (int): Number of disk-cache shards for concurrent file-cache access. Defaults to 32.
+            Each shard is a separate on-disk store that holds open file handles, so a higher value
+            raises the process's file-descriptor count — which can exceed a low ``ulimit -n`` and
+            surface as "Too many open files". 32 comfortably covers the default ``maxWorkers`` of 25.
+            Must be positive. Immutable at runtime — set it via the ``RAPIDATA_cacheShards`` environment
+            variable. Only used for file uploads when cacheToDisk=True.
         enableBatchUpload (bool): Enable batch URL uploading (two-step process). Defaults to True.
         batchSize (int): Number of URLs per batch (100-5000). Defaults to 1000.
         batchPollInterval (float): Polling interval in seconds. Defaults to 0.5.
@@ -134,8 +137,9 @@ def _apply_env_vars(cls, data: Any) -> Any:
         frozen=True,
     )
     cacheShards: int = Field(
-        default=128,
+        default=32,
         frozen=True,
+        description="Disk-cache shards. Each opens file handles; keep low enough for ulimit -n.",
     )
     batchSize: int = Field(
         default=1000,

src/rapidata/rapidata_client/exceptions/failed_upload_exception.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+import errno
 from rapidata.rapidata_client.datapoints._datapoint import Datapoint
 from .failed_upload import FailedUpload
 from typing import TYPE_CHECKING, Optional
@@ -95,6 +96,20 @@ def failures_by_reason(self) -> dict[str, list[Datapoint]]:
             grouped[failed_upload.error_message].append(failed_upload.item)
         return dict(grouped)
 
+    def _has_fd_exhaustion_failure(self) -> bool:
+        """Whether any failure looks like the OS running out of file descriptors.
+
+        Checks the errno first (EMFILE), falling back to the message text for
+        cases where the OSError was wrapped or stringified before it reached us.
+        """
+        for fu in self._failed_uploads:
+            exc = fu.exception
+            if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+                return True
+            if "too many open files" in fu.error_message.lower():
+                return True
+        return False
+
     def __str__(self) -> str:
         total = len(self._failed_uploads)
         if total == 0:
@@ -119,6 +134,16 @@ def __str__(self) -> str:
             lines.append("  ]")
 
         failed_upload_message = "\n".join(lines)
+        if self._has_fd_exhaustion_failure():
+            failed_upload_message += (
+                "\n\nThese failures look like file-descriptor exhaustion ('Too many open "
+                "files'). The upload cache and worker pool each hold open file handles, so a "
+                "large or concurrent upload can exceed a low 'ulimit -n'. To fix, either raise "
+                "the OS limit (e.g. 'ulimit -n 8192') or lower the SDK's footprint via the "
+                "'RAPIDATA_cacheShards' (default 32) and 'RAPIDATA_maxWorkers' (default 25) "
+                "environment variables. See "
+                "https://docs.rapidata.ai/3.x/config/#upload-configuration-options"
+            )
         if self.machine is not None:
             failed_upload_message += (
                 "\n\nNo job definition was created because the upload stayed outside the "

tests/rapidata_client/exceptions/test_failed_upload_exception.py

@@ -0,0 +1,51 @@
+"""Tests for FailedUploadException message rendering.
+
+Focused on the file-descriptor-exhaustion hint: when an upload fails because the
+process ran out of file descriptors, the exception message must point the user
+at the knobs (cacheShards / maxWorkers / ulimit) rather than only listing the
+raw OSError.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+from rapidata.rapidata_client.exceptions.failed_upload import FailedUpload
+from rapidata.rapidata_client.exceptions.failed_upload_exception import (
+    FailedUploadException,
+)
+
+
+def _exception(failed_uploads: list[FailedUpload]) -> FailedUploadException:
+    return FailedUploadException(dataset=MagicMock(), failed_uploads=failed_uploads)
+
+
+def test_hint_added_when_errno_is_emfile() -> None:
+    exc = OSError(24, "Too many open files")
+    failure = FailedUpload.from_exception("img.jpg", exc)
+
+    message = str(_exception([failure]))
+
+    assert "file-descriptor exhaustion" in message
+    assert "RAPIDATA_cacheShards" in message
+    assert "RAPIDATA_maxWorkers" in message
+    assert "ulimit -n" in message
+
+
+def test_hint_added_when_only_message_matches() -> None:
+    # Some layers stringify the OSError before it reaches us, losing the errno.
+    failure = FailedUpload(
+        item="img.jpg",
+        error_message="[Errno 24] Too many open files: 'img.jpg'",
+        error_type="OSError",
+    )
+
+    assert "file-descriptor exhaustion" in str(_exception([failure]))
+
+
+def test_no_hint_for_unrelated_failures() -> None:
+    failure = FailedUpload(
+        item="img.jpg", error_message="404 not found", error_type="RapidataError"
+    )
+
+    assert "file-descriptor exhaustion" not in str(_exception([failure]))