feat: enable multi-datacenter support (#266)

* feat: enable multi-datacenter support

* docs: update SDK reference and deploy guide for multi-datacenter support

* feat: add datacenter alias on NetworkVolume

* fix: pass locations through manifest and resource provisioner

* fix: remove dead _is_prod_environment function

* fix: address copilot review on multi-volume support

* fix: address PR review comments for multi-datacenter support

* style: fix ruff formatting in manifest.py

* fix: remove redundant type annotation on _deployed_volume_ids reset

* fix: align datacenter enum to s3-enabled DCs

* fix: address post-rebase review comments

* fix: address PR review findings for multi-datacenter support

- fix US_GA_1 references to US_GA_2 in docs, tests (enum member does not exist)
- add field_serializer for datacenter on ServerlessResource for consistency
- strip runtime-assigned volume IDs from config_hash to prevent false drift
- add tests for multi-volume manifest serialization
- add tests for multi-volume provisioner reconstruction
- add test verifying volume id does not affect config hash

Author: KAJdev

docs/Flash_Deploy_Guide.md

@@ -289,13 +289,14 @@ result = await vllm.post("/v1/completions", {"prompt": "hello"})
 ### Persistent Storage
 
 ```python
-from runpod_flash import Endpoint, GpuGroup, NetworkVolume
+from runpod_flash import Endpoint, GpuGroup, DataCenter, NetworkVolume, PodTemplate
 
-vol = NetworkVolume(id="vol_abc123")
+vol = NetworkVolume(name="model-cache", size=100, datacenter=DataCenter.US_GA_1)
 
 @Endpoint(
     name="model-server",
     gpu=GpuGroup.AMPERE_80,
+    datacenter=DataCenter.US_GA_2,
     volume=vol,
     template=PodTemplate(containerDiskInGb=100),
 )
@@ -304,6 +305,24 @@ async def serve(data: dict) -> dict:
     ...
 ```
 
+Multiple volumes across datacenters:
+
+```python
+volumes = [
+    NetworkVolume(name="models-us", size=100, datacenter=DataCenter.US_GA_1),
+    NetworkVolume(name="models-eu", size=100, datacenter=DataCenter.EU_RO_1),
+]
+
+@Endpoint(
+    name="global-server",
+    gpu=GpuGroup.AMPERE_80,
+    datacenter=[DataCenter.US_GA_1, DataCenter.EU_RO_1],
+    volume=volumes,
+)
+async def serve(data: dict) -> dict:
+    ...
+```
+
 ## Troubleshooting
 
 ### Build Issues

docs/Flash_SDK_Reference.md

@@ -20,8 +20,8 @@ Endpoint(
     dependencies: Optional[List[str]] = None,
     system_dependencies: Optional[List[str]] = None,
     accelerate_downloads: bool = True,
-    volume: Optional[NetworkVolume] = None,
-    datacenter: DataCenter = DataCenter.EU_RO_1,
+    volume: Optional[Union[NetworkVolume, List[NetworkVolume]]] = None,
+    datacenter: Optional[Union[DataCenter, List[DataCenter], str, List[str]]] = None,
     env: Optional[Dict[str, str]] = None,
     gpu_count: int = 1,
     execution_timeout_ms: int = 0,
@@ -46,8 +46,8 @@ Endpoint(
 | `dependencies` | `list[str]` | `None` | Python packages to install (e.g., `["torch", "numpy==1.24"]`). |
 | `system_dependencies` | `list[str]` | `None` | System packages to install. |
 | `accelerate_downloads` | `bool` | `True` | Enable accelerated downloads. |
-| `volume` | `NetworkVolume` | `None` | Network volume for persistent storage. |
-| `datacenter` | `DataCenter` | `EU_RO_1` | Preferred datacenter. |
+| `volume` | `NetworkVolume` or list | `None` | Network volume(s) for persistent storage. One volume per datacenter. |
+| `datacenter` | `DataCenter`, list, `str`, or `None` | `None` | Datacenter(s) to deploy into. `None` means all available DCs. Accepts a single value, a list, or string DC IDs. CPU endpoints must use DCs in `CPU_DATACENTERS`. |
 | `env` | `dict[str, str]` | `None` | Environment variables for the endpoint. |
 | `gpu_count` | `int` | `1` | GPUs per worker. |
 | `execution_timeout_ms` | `int` | `0` | Max execution time in ms. 0 = no limit. |
@@ -335,8 +335,25 @@ CPU instance selection. Can also be passed as a string to `cpu=`.
 
 | Value | Location |
 |-------|----------|
-| `DataCenter.EU_RO_1` | Europe - Romania (default) |
-| `DataCenter.US_TX_3` | US - Texas |
+| `DataCenter.US_CA_2` | US - California |
+| `DataCenter.US_GA_2` | US - Georgia |
+| `DataCenter.US_IL_1` | US - Illinois |
+| `DataCenter.US_KS_2` | US - Kansas |
+| `DataCenter.US_MD_1` | US - Maryland |
+| `DataCenter.US_MO_1` | US - Missouri |
+| `DataCenter.US_MO_2` | US - Missouri |
+| `DataCenter.US_NC_1` | US - North Carolina |
+| `DataCenter.US_NC_2` | US - North Carolina |
+| `DataCenter.US_NE_1` | US - Nebraska |
+| `DataCenter.US_WA_1` | US - Washington |
+| `DataCenter.EU_CZ_1` | Europe - Czech Republic |
+| `DataCenter.EU_RO_1` | Europe - Romania |
+| `DataCenter.EUR_IS_1` | Europe - Iceland |
+| `DataCenter.EUR_NO_1` | Europe - Norway |
+
+When `datacenter=None` (the default), the endpoint is available in all data centers.
+
+CPU endpoints are restricted to the `CPU_DATACENTERS` subset: `EU_RO_1`.
 
 ### CudaVersion
 
@@ -350,12 +367,22 @@ CPU instance selection. Can also be passed as a string to `cpu=`.
 
 ### NetworkVolume
 
-Persistent storage that survives worker restarts.
+Persistent storage that survives worker restarts. Each volume is tied to a specific datacenter.
 
 ```python
-from runpod_flash import NetworkVolume
+from runpod_flash import NetworkVolume, DataCenter
 
+# existing volume by ID
 vol = NetworkVolume(id="vol_abc123")
+
+# create a new volume in a specific datacenter
+vol = NetworkVolume(name="my-models", size=100, datacenter=DataCenter.US_GA_2)
+
+# multiple volumes across datacenters (one per DC)
+volumes = [
+    NetworkVolume(name="models-us", size=100, datacenter=DataCenter.US_GA_1),
+    NetworkVolume(name="models-eu", size=100, datacenter=DataCenter.EU_RO_1),
+]
 ```
 
 ### PodTemplate
@@ -392,6 +419,7 @@ from runpod_flash import (
     CpuInstanceType,
     CudaVersion,
     DataCenter,
+    CPU_DATACENTERS,
     NetworkVolume,
     PodTemplate,
     ServerlessScalerType,

src/runpod_flash/__init__.py

@@ -17,6 +17,7 @@
     from .client import remote
     from .endpoint import Endpoint, EndpointJob
     from .core.resources import (
+        CPU_DATACENTERS,
         CpuInstanceType,
         CpuLiveLoadBalancer,
         CpuLiveServerless,
@@ -58,6 +59,7 @@
 
 _RESOURCE_NAMES = frozenset(
     {
+        "CPU_DATACENTERS",
         "CpuInstanceType",
         "CpuLiveLoadBalancer",
         "CpuLiveServerless",
@@ -104,6 +106,7 @@ def __getattr__(name):
         return remote
     elif name in _RESOURCE_NAMES:
         from .core.resources import (
+            CPU_DATACENTERS,
             CpuInstanceType,
             CpuLiveLoadBalancer,
             CpuLiveServerless,
@@ -126,6 +129,7 @@ def __getattr__(name):
         )
 
         attrs = {
+            "CPU_DATACENTERS": CPU_DATACENTERS,
             "CpuInstanceType": CpuInstanceType,
             "CpuLiveLoadBalancer": CpuLiveLoadBalancer,
             "CpuLiveServerless": CpuLiveServerless,
@@ -173,6 +177,7 @@ def __getattr__(name):
     "Endpoint",
     "EndpointJob",
     "remote",
+    "CPU_DATACENTERS",
     "CpuInstanceType",
     "CpuLiveLoadBalancer",
     "CpuLiveServerless",

src/runpod_flash/cli/commands/build_utils/manifest.py

@@ -25,6 +25,24 @@
 RESERVED_PATHS = ["/execute", "/ping"]
 
 
+def _serialize_network_volume(nv) -> dict:
+    """Serialize a NetworkVolume to a manifest-safe dict."""
+    nv_config: dict = {}
+    if nv.name is not None:
+        nv_config["name"] = nv.name
+    if getattr(nv, "id", None) is not None:
+        nv_config["id"] = nv.id
+    if nv.size is not None:
+        nv_config["size"] = nv.size
+    if hasattr(nv, "dataCenterId") and nv.dataCenterId is not None:
+        nv_config["dataCenterId"] = (
+            nv.dataCenterId.value
+            if hasattr(nv.dataCenterId, "value")
+            else nv.dataCenterId
+        )
+    return nv_config
+
+
 @dataclass
 class ManifestFunction:
     """Function entry in manifest."""
@@ -218,24 +236,29 @@ def _extract_config_properties(config: Dict[str, Any], resource_config) -> None:
         ):
             config["scalerValue"] = resource_config.scalerValue
 
+        if hasattr(resource_config, "locations") and resource_config.locations:
+            config["locations"] = resource_config.locations
+
         if hasattr(resource_config, "env") and resource_config.env:
             env_dict = dict(resource_config.env)
             env_dict.pop("RUNPOD_API_KEY", None)
             if env_dict:
                 config["env"] = env_dict
 
-        if hasattr(resource_config, "networkVolume") and resource_config.networkVolume:
-            nv = resource_config.networkVolume
-            nv_config = {"name": nv.name}
-            if nv.size is not None:
-                nv_config["size"] = nv.size
-            if hasattr(nv, "dataCenterId") and nv.dataCenterId is not None:
-                nv_config["dataCenterId"] = (
-                    nv.dataCenterId.value
-                    if hasattr(nv.dataCenterId, "value")
-                    else nv.dataCenterId
-                )
-            config["networkVolume"] = nv_config
+        if (
+            hasattr(resource_config, "networkVolumes")
+            and resource_config.networkVolumes
+        ):
+            config["networkVolumes"] = [
+                _serialize_network_volume(nv) for nv in resource_config.networkVolumes
+            ]
+
+        elif (
+            hasattr(resource_config, "networkVolume") and resource_config.networkVolume
+        ):
+            config["networkVolume"] = _serialize_network_volume(
+                resource_config.networkVolume
+            )
 
         elif (
             hasattr(resource_config, "networkVolumeId")

src/runpod_flash/core/api/runpod.py

@@ -273,6 +273,10 @@ async def save_endpoint(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
                 locations
                 name
                 networkVolumeId
+                networkVolumeIds {
+                    networkVolumeId
+                    dataCenterId
+                }
                 flashEnvironmentId
                 scalerType
                 scalerValue

src/runpod_flash/core/resources/__init__.py

@@ -18,7 +18,7 @@
 )
 from .serverless_cpu import CpuServerlessEndpoint
 from .template import PodTemplate
-from .network_volume import NetworkVolume, DataCenter
+from .network_volume import NetworkVolume, DataCenter, CPU_DATACENTERS
 from .load_balancer_sls_resource import (
     CpuLoadBalancerSlsResource,
     LoadBalancerSlsResource,
@@ -33,6 +33,7 @@
     "CpuLoadBalancerSlsResource",
     "CpuServerlessEndpoint",
     "CudaVersion",
+    "CPU_DATACENTERS",
     "DataCenter",
     "DeployableResource",
     "GpuGroup",

src/runpod_flash/core/resources/load_balancer_sls_resource.py

@@ -342,6 +342,7 @@ class CpuLoadBalancerSlsResource(CpuEndpointMixin, LoadBalancerSlsResource):
         "allowedCudaVersions",
         "imageName",
         "networkVolume",
+        "networkVolumes",
         "python_version",
     }
 
@@ -401,6 +402,7 @@ def config_hash(self) -> str:
             "flashEnvironmentId",
             "imageName",
             "networkVolume",
+            "networkVolumes",
             "instanceIds",  # CPU-specific
             "workersMin",  # Scaling
             "workersMax",

src/runpod_flash/core/resources/network_volume.py

@@ -6,6 +6,7 @@
 from pydantic import (
     Field,
     field_serializer,
+    model_validator,
 )
 
 from ..api.runpod import RunpodRestClient
@@ -17,12 +18,50 @@
 
 
 class DataCenter(str, Enum):
-    """
-    Enum representing available data centers for network volumes.
-    #TODO: Add more data centers as needed. Lock this to the available data center.
-    """
-
+    """Enum representing available RunPod data centers."""
+
+    # north america
+    US_CA_2 = "US-CA-2"
+    US_GA_2 = "US-GA-2"
+    US_IL_1 = "US-IL-1"
+    US_KS_2 = "US-KS-2"
+    US_MD_1 = "US-MD-1"
+    US_MO_1 = "US-MO-1"
+    US_MO_2 = "US-MO-2"
+    US_NC_1 = "US-NC-1"
+    US_NC_2 = "US-NC-2"
+    US_NE_1 = "US-NE-1"
+    US_WA_1 = "US-WA-1"
+
+    # europe
+    EU_CZ_1 = "EU-CZ-1"
     EU_RO_1 = "EU-RO-1"
+    EUR_IS_1 = "EUR-IS-1"
+    EUR_NO_1 = "EUR-NO-1"
+
+    @classmethod
+    def from_string(cls, value: str) -> "DataCenter":
+        """Parse a datacenter ID string into a DataCenter enum.
+
+        Accepts the canonical form (e.g. "EU-RO-1") as well as common
+        variations like lowercase or underscore-separated.
+        """
+        normalized = value.strip().upper().replace("_", "-")
+        try:
+            return cls(normalized)
+        except ValueError:
+            valid = ", ".join(dc.value for dc in cls)
+            raise ValueError(
+                f"Unknown datacenter '{value}'. Valid datacenters: {valid}"
+            )
+
+
+# data centers that support CPU serverless endpoints
+CPU_DATACENTERS: frozenset[DataCenter] = frozenset(
+    {
+        DataCenter.EU_RO_1,
+    }
+)
 
 
 class NetworkVolume(DeployableResource):
@@ -41,22 +80,42 @@ class NetworkVolume(DeployableResource):
         "name",
     }
 
-    # Internal fixed value
-    dataCenterId: DataCenter = Field(default=DataCenter.EU_RO_1, frozen=True)
+    # public alias -- users pass datacenter=, which syncs to dataCenterId for the API
+    datacenter: Optional[DataCenter] = Field(default=None, exclude=True)
+    dataCenterId: DataCenter = Field(default=DataCenter.EU_RO_1)
 
     id: Optional[str] = Field(default=None)
-    name: str
+    name: Optional[str] = None
     size: Optional[int] = Field(default=100, gt=0)  # Size in GB
 
+    @model_validator(mode="before")
+    @classmethod
+    def sync_datacenter_alias(cls, data):
+        """Allow datacenter= as a user-friendly alias for dataCenterId."""
+        if isinstance(data, dict):
+            dc = data.pop("datacenter", None)
+            if dc is not None and "dataCenterId" not in data:
+                data["dataCenterId"] = dc
+        return data
+
+    @model_validator(mode="after")
+    def require_name_or_id(self):
+        """Either name or id must be provided."""
+        if not self.name and not self.id:
+            raise ValueError("either 'name' or 'id' must be provided")
+        return self
+
     def __str__(self) -> str:
         return f"{self.__class__.__name__}:{self.id}"
 
     @property
     def resource_id(self) -> str:
         """Unique resource ID based on name and datacenter for idempotent behavior."""
-        # Use name + datacenter to ensure idempotence
         resource_type = self.__class__.__name__
-        config_key = f"{self.name}:{self.dataCenterId.value}"
+        if self.name:
+            config_key = f"{self.name}:{self.dataCenterId.value}"
+        else:
+            config_key = f"id:{self.id}"
         hash_obj = hashlib.md5(f"{resource_type}:{config_key}".encode())
         return f"{resource_type}_{hash_obj.hexdigest()}"