feat: implement lazy file loading to prevent 'too many open files' errors

- Add FileDescriptorManager singleton to track and limit open file descriptors
- Implement LazyFileLoader class that opens files only when needed for reading
- Add configurable max_open_files parameter to fullscans.post() and diffscans.create_from_repo()
- Auto-close files when fully read and use LRU eviction when limit reached
- Add comprehensive documentation with v3.0 migration notes
- Maintain backward compatibility with use_lazy_loading=False default
- Support cross-platform operation (Unix/Linux/macOS/Windows)
- Include retry logic with garbage collection for edge cases

This prevents file descriptor exhaustion when uploading large numbers of manifest files (e.g., 1956 files) on systems with low ulimit values.

Author: dacoburn

.gitignore

@@ -12,4 +12,5 @@ dist
 *.build
 *.dist
 *.egg-info
-*.cpython-312.pyc
+*.cpython-312.pyc
+example-socket-export.py

socketdev/dependencies/__init__.py

@@ -2,6 +2,7 @@
 from urllib.parse import urlencode
 import logging
 from socketdev.tools import load_files
+from ..utils import Utils
 
 log = logging.getLogger("socketdev")
 
@@ -12,9 +13,13 @@ class Dependencies:
     def __init__(self, api):
         self.api = api
 
-    def post(self, files: list, params: dict) -> dict:
-        loaded_files = []
-        loaded_files = load_files(files, loaded_files)
+    def post(self, files: list, params: dict, use_lazy_loading: bool = False, workspace: str = None) -> dict:
+        if use_lazy_loading:
+            loaded_files = Utils.load_files_for_sending_lazy(files, workspace)
+        else:
+            loaded_files = []
+            loaded_files = load_files(files, loaded_files)
+        
         path = "dependencies/upload?" + urlencode(params)
         response = self.api.do_request(path=path, files=loaded_files, method="POST")
         if response.status_code == 200:

socketdev/diffscans/__init__.py

@@ -1,6 +1,7 @@
 import json
 import logging
 from typing import Any, Dict, Optional, Union
+from ..utils import Utils
 
 log = logging.getLogger("socketdev")
 
@@ -29,13 +30,44 @@ def get(self, org_slug: str, diff_scan_id: str) -> dict:
         log.error(f"Error fetching diff scan: {response.status_code}, message: {response.text}")
         return {}
 
-    def create_from_repo(self, org_slug: str, repo_slug: str, files: list, params: Optional[Dict[str, Any]] = None) -> dict:
-        """Create a diff scan from repo HEAD, uploading files as multipart form data."""
+    def create_from_repo(self, org_slug: str, repo_slug: str, files: list, params: Optional[Dict[str, Any]] = None, use_lazy_loading: bool = False, workspace: str = None, max_open_files: int = 100) -> dict:
+        """
+        Create a diff scan from repo HEAD, uploading files as multipart form data.
+        
+        Args:
+            org_slug: Organization slug
+            repo_slug: Repository slug  
+            files: List of file paths to upload for scanning
+            params: Optional query parameters for the request
+            use_lazy_loading: Whether to use lazy file loading to prevent "too many open files" 
+                            errors when uploading large numbers of files (default: False)
+                            NOTE: In version 3.0, this will default to True for better performance
+            workspace: Base directory path to make file paths relative to
+            max_open_files: Maximum number of files to keep open simultaneously when using 
+                          lazy loading. Useful for systems with low ulimit values (default: 100)
+        
+        Returns:
+            dict: API response containing diff scan results
+            
+        Note:
+            When use_lazy_loading=True, files are opened only when needed during upload,
+            preventing file descriptor exhaustion. The max_open_files parameter controls how many
+            files can be open simultaneously - set this lower on systems with restrictive ulimits.
+            
+            For large file uploads (>100 files), it's recommended to set use_lazy_loading=True.
+        """
         import urllib.parse
         path = f"orgs/{org_slug}/diff-scans/from-repo/{repo_slug}"
         if params:
             path += "?" + urllib.parse.urlencode(params)
-        response = self.api.do_request(path=path, method="POST", files=files)
+        
+        # Use lazy loading if requested
+        if use_lazy_loading:
+            prepared_files = Utils.load_files_for_sending_lazy(files, workspace, max_open_files)
+        else:
+            prepared_files = files
+        
+        response = self.api.do_request(path=path, method="POST", files=prepared_files)
         if response.status_code in (200, 201):
             return response.json()
         log.error(f"Error creating diff scan from repo: {response.status_code}, message: {response.text}")

socketdev/fullscans/__init__.py

@@ -720,15 +720,45 @@ def get(self, org_slug: str, params: dict, use_types: bool = False) -> Union[dic
             )
         return {}
 
-    def post(self, files: list, params: FullScanParams, use_types: bool = False) -> Union[dict, CreateFullScanResponse]:
+    def post(self, files: list, params: FullScanParams, use_types: bool = False, use_lazy_loading: bool = False, workspace: str = None, max_open_files: int = 100) -> Union[dict, CreateFullScanResponse]:
+        """
+        Create a new full scan by uploading manifest files.
+        
+        Args:
+            files: List of file paths to upload for scanning
+            params: FullScanParams object containing scan configuration
+            use_types: Whether to return typed response objects (default: False)
+            use_lazy_loading: Whether to use lazy file loading to prevent "too many open files" 
+                            errors when uploading large numbers of files (default: False)
+                            NOTE: In version 3.0, this will default to True for better performance
+            workspace: Base directory path to make file paths relative to
+            max_open_files: Maximum number of files to keep open simultaneously when using 
+                          lazy loading. Useful for systems with low ulimit values (default: 100)
+        
+        Returns:
+            dict or CreateFullScanResponse: API response containing scan results
+            
+        Note:
+            When use_lazy_loading=True, files are opened only when needed during upload,
+            preventing file descriptor exhaustion. The max_open_files parameter controls how many
+            files can be open simultaneously - set this lower on systems with restrictive ulimits.
+            
+            For large file uploads (>100 files), it's recommended to set use_lazy_loading=True.
+        """
         Utils.validate_integration_type(params.integration_type if params.integration_type else "api")
         org_slug = str(params.org_slug)
         params_dict = params.to_dict()
         params_dict.pop("org_slug")
         params_arg = urllib.parse.urlencode(params_dict)
         path = "orgs/" + org_slug + "/full-scans?" + str(params_arg)
 
-        response = self.api.do_request(path=path, method="POST", files=files)
+        # Use lazy loading if requested
+        if use_lazy_loading:
+            prepared_files = Utils.load_files_for_sending_lazy(files, workspace, max_open_files)
+        else:
+            prepared_files = files
+
+        response = self.api.do_request(path=path, method="POST", files=prepared_files)
 
         if response.status_code == 201:
             result = response.json()