feat(openai): route OpenAI Realtime (WebSocket) through LLM Gateway

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes to `uipath_llm_client` (core package) will be documented in this file.
 
+## [1.15.0] - 2026-06-18
+
+### Added
+- **OpenAI Realtime (WebSocket) routed through the LLM Gateway.** `UiPathOpenAI` and `UiPathAsyncOpenAI` now expose `client.realtime.connect()`, exactly like the stock OpenAI SDK, opening a WebSocket to the gateway's passthrough realtime endpoint (`.../vendor/<vendor>/model/<model>/realtime`). On connect the client points its `websocket_base_url` at the gateway, refreshes the S2S bearer token into `api_key` (sent as `Authorization: Bearer` on the WebSocket upgrade), and injects the `X-UiPath-*` routing headers. The realtime URL uses the `nativeopenai` vendor segment and is built lazily on `.realtime` access, so completions/embeddings construction and auth are unaffected. New helper `build_realtime_ws_base_url(settings, model_name=..., vendor_type=...)` in `uipath.llm_client.clients.openai.realtime`.
+- The `openai` optional extra now installs `openai[realtime]`, pulling in the `websockets` dependency required for realtime connections.
+
 ## [1.14.0] - 2026-06-15
 
 ### Added

README.md

@@ -466,6 +466,35 @@ All native SDK wrappers are available in sync and async variants:
 | `UiPathAnthropicFoundry` / `UiPathAsyncAnthropicFoundry` | `anthropic.AnthropicFoundry` | Anthropic via Azure Foundry |
 | `UiPathGoogle` | `google.genai.Client` | Google Gemini models |
 
+#### Realtime (WebSocket)
+
+`UiPathOpenAI` / `UiPathAsyncOpenAI` also expose the OpenAI Realtime API over a WebSocket, routed through the LLM Gateway. Use `client.realtime.connect()` exactly as with the stock OpenAI SDK — the bearer token and `X-UiPath-*` routing headers are applied on the WebSocket upgrade automatically. Requires the `openai` extra (it pulls in `websockets`).
+
+```python
+import asyncio
+from uipath.llm_client.clients.openai import UiPathAsyncOpenAI
+
+async def main():
+    client = UiPathAsyncOpenAI(model_name="gpt-realtime")
+    async with client.realtime.connect() as conn:
+        await conn.session.update(session={"type": "realtime", "output_modalities": ["text"]})
+        await conn.conversation.item.create(
+            item={
+                "type": "message",
+                "role": "user",
+                "content": [{"type": "input_text", "text": "Say hello!"}],
+            }
+        )
+        await conn.response.create()
+        async for event in conn:
+            if event.type == "response.output_text.delta":
+                print(event.delta, end="", flush=True)
+            elif event.type == "response.done":
+                break
+
+asyncio.run(main())
+```
+
 ### Low-Level HTTP Client
 
 For completely custom HTTP requests, use the low-level HTTPX client directly:

packages/uipath_langchain_client/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes to `uipath_langchain_client` will be documented in this file.
 
+## [1.15.0] - 2026-06-18
+
+### Changed
+- Bumped `uipath-llm-client` floor to `>=1.15.0` to pick up OpenAI Realtime (WebSocket) support on `UiPathOpenAI` / `UiPathAsyncOpenAI` (`client.realtime.connect()` routed through the LLM Gateway). The realtime clients live in the core package (`uipath.llm_client.clients.openai`); no LangChain-specific wrapper is added, since LangChain has no realtime chat-model abstraction — realtime is used by dropping down to the core client.
+- The `openai` extra now also installs `uipath-llm-client[openai]` (which pulls `openai[realtime]` → `websockets`), so realtime works out of the box from a `uipath-langchain-client[openai]` install.
+
 ## [1.14.0] - 2026-06-15
 
 ### Added

packages/uipath_langchain_client/pyproject.toml

@@ -6,12 +6,16 @@ readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
     "langchain>=1.2.15,<2.0.0",
-    "uipath-llm-client>=1.14.0,<2.0.0",
+    "uipath-llm-client>=1.15.0,<2.0.0",
 ]
 
 [project.optional-dependencies]
 openai = [
     "langchain-openai>=1.2.0,<2.0.0",
+    # Pulls the core OpenAI extra (openai[realtime] -> websockets) so realtime is
+    # usable via uipath.llm_client.clients.openai.UiPathAsyncOpenAI from a
+    # langchain install. LangChain itself has no realtime chat-model abstraction.
+    "uipath-llm-client[openai]>=1.15.0,<2.0.0",
 ]
 google = [
     "langchain-google-genai>=4.2.2,<5.0.0",

packages/uipath_langchain_client/src/uipath_langchain_client/__version__.py

@@ -1,3 +1,3 @@
 __title__ = "UiPath LangChain Client"
 __description__ = "A Python client for interacting with UiPath's LLM services via LangChain."
-__version__ = "1.14.0"
+__version__ = "1.15.0"

pyproject.toml

@@ -19,7 +19,7 @@ authors = [
 
 [project.optional-dependencies]
 openai = [
-    "openai>=2.30.0,<3.0.0",
+    "openai[realtime]>=2.30.0,<3.0.0",
 ]
 google = [
     "google-genai>=1.73.1,<2.0.0",

src/uipath/llm_client/__version__.py

@@ -1,3 +1,3 @@
 __title__ = "UiPath LLM Client"
 __description__ = "A Python client for interacting with UiPath's LLM services."
-__version__ = "1.14.0"
+__version__ = "1.15.0"

src/uipath/llm_client/clients/openai/client.py

@@ -1,13 +1,20 @@
 import logging
 from collections.abc import Mapping, Sequence
+from functools import cached_property
 
+from uipath.llm_client.clients.openai.realtime import (
+    DEFAULT_REALTIME_VENDOR,
+    _UiPathAsyncRealtime,
+    _UiPathRealtime,
+)
 from uipath.llm_client.clients.openai.utils import OpenAIRequestHandler
 from uipath.llm_client.httpx_client import UiPathHttpxAsyncClient, UiPathHttpxClient
 from uipath.llm_client.settings import UiPathBaseSettings, get_default_client_settings
 from uipath.llm_client.utils.retry import RetryConfig
 
 try:
     from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
+    from openai.resources.realtime import AsyncRealtime, Realtime
 except ImportError as e:
     raise ImportError(
         "The 'openai' extra is required to use UiPathOpenAIClient. "
@@ -74,6 +81,24 @@ def __init__(
             http_client=httpx_client,
             base_url=str(httpx_client.base_url).rstrip("/"),
         )
+        self._uipath_client_settings = client_settings
+        self._uipath_realtime_model = model_name
+
+    # Subtype override of the SDK's cached_property; returns a gateway-routed
+    # Realtime resource. pyright flags any cached_property override, so suppress.
+    @cached_property
+    def realtime(self) -> Realtime:  # pyright: ignore[reportIncompatibleMethodOverride]
+        """OpenAI Realtime (WebSocket) resource routed through the LLM Gateway.
+
+        Use ``with client.realtime.connect() as conn:`` — the connection targets
+        the gateway's passthrough realtime endpoint for this client's model.
+        """
+        return _UiPathRealtime(
+            self,
+            settings=self._uipath_client_settings,
+            model=self._uipath_realtime_model,
+            vendor_type=DEFAULT_REALTIME_VENDOR,
+        )
 
 
 class UiPathAsyncOpenAI(AsyncOpenAI):
@@ -135,6 +160,24 @@ def __init__(
             http_client=httpx_client,
             base_url=str(httpx_client.base_url).rstrip("/"),
         )
+        self._uipath_client_settings = client_settings
+        self._uipath_realtime_model = model_name
+
+    # Subtype override of the SDK's cached_property; returns a gateway-routed
+    # AsyncRealtime resource. pyright flags any cached_property override, so suppress.
+    @cached_property
+    def realtime(self) -> AsyncRealtime:  # pyright: ignore[reportIncompatibleMethodOverride]
+        """OpenAI Realtime (WebSocket) resource routed through the LLM Gateway.
+
+        Use ``async with client.realtime.connect() as conn:`` — the connection
+        targets the gateway's passthrough realtime endpoint for this client's model.
+        """
+        return _UiPathAsyncRealtime(
+            self,
+            settings=self._uipath_client_settings,
+            model=self._uipath_realtime_model,
+            vendor_type=DEFAULT_REALTIME_VENDOR,
+        )
 
 
 class UiPathAzureOpenAI(AzureOpenAI):

src/uipath/llm_client/clients/openai/realtime.py

@@ -0,0 +1,214 @@
+"""Realtime (WebSocket) support for the UiPath OpenAI clients.
+
+The OpenAI Realtime API speaks over a WebSocket rather than HTTP, so it does not
+go through the httpx routing used for completions/embeddings. Instead,
+``UiPathOpenAI`` / ``UiPathAsyncOpenAI`` expose ``client.realtime.connect()`` —
+exactly like the stock OpenAI SDK — by swapping in the resource wrappers defined
+here. On connect these wrappers:
+
+- point the client's ``websocket_base_url`` at the gateway passthrough realtime
+  path (``.../vendor/<vendor>/model/<model>``); the SDK appends ``/realtime``,
+- set the S2S bearer token as ``api_key`` (the SDK sends it as
+  ``Authorization: Bearer`` on the WebSocket upgrade),
+- inject the ``X-UiPath-*`` routing headers on the upgrade request.
+
+Completions/embeddings are unaffected: their auth comes from the httpx auth
+pipeline, and the realtime URL is only built when ``.realtime`` is accessed.
+
+Example:
+    >>> import asyncio
+    >>> from uipath.llm_client.clients.openai import UiPathAsyncOpenAI
+    >>>
+    >>> async def main():
+    ...     client = UiPathAsyncOpenAI(model_name="gpt-realtime")
+    ...     async with client.realtime.connect() as conn:
+    ...         await conn.session.update(
+    ...             session={"type": "realtime", "output_modalities": ["text"]}
+    ...         )
+    ...         await conn.conversation.item.create(
+    ...             item={
+    ...                 "type": "message",
+    ...                 "role": "user",
+    ...                 "content": [{"type": "input_text", "text": "Say hello!"}],
+    ...             }
+    ...         )
+    ...         await conn.response.create()
+    ...         async for event in conn:
+    ...             if event.type == "response.output_text.delta":
+    ...                 print(event.delta, end="")
+    ...             elif event.type == "response.done":
+    ...                 break
+    >>> asyncio.run(main())
+"""
+
+import re
+
+from typing_extensions import override
+
+from uipath.llm_client.settings import UiPathAPIConfig, UiPathBaseSettings
+from uipath.llm_client.settings.constants import RoutingMode
+from uipath.llm_client.settings.llmgateway.auth import LLMGatewayS2SAuth
+
+try:
+    from openai import AsyncOpenAI, OpenAI
+    from openai._types import Headers, Omit, Query, omit
+    from openai.resources.realtime import AsyncRealtime, Realtime
+    from openai.resources.realtime.realtime import (
+        AsyncRealtimeConnectionManager,
+        RealtimeConnectionManager,
+    )
+    from openai.types.websocket_connection_options import WebSocketConnectionOptions
+except ImportError as e:
+    raise ImportError(
+        "The 'openai' extra is required for realtime support. "
+        "Install it with: uv add uipath-llm-client[openai]"
+    ) from e
+
+# The gateway expects the native-OpenAI realtime vendor segment in the path.
+DEFAULT_REALTIME_VENDOR = "nativeopenai"
+# Passthrough api_type segment for the realtime endpoint (not a normalized ApiType).
+REALTIME_API_TYPE = "realtime"
+
+
+def _realtime_api_config(vendor_type: str) -> UiPathAPIConfig:
+    return UiPathAPIConfig(
+        api_type=REALTIME_API_TYPE,
+        routing_mode=RoutingMode.PASSTHROUGH,
+        vendor_type=vendor_type,
+    )
+
+
+def build_realtime_ws_base_url(
+    settings: UiPathBaseSettings,
+    *,
+    model_name: str,
+    vendor_type: str = DEFAULT_REALTIME_VENDOR,
+) -> str:
+    """Build the ``websocket_base_url`` to hand to the OpenAI SDK.
+
+    The SDK appends ``/realtime`` to ``websocket_base_url`` when connecting, so
+    this strips the trailing ``/realtime`` produced by ``build_base_url`` and
+    converts the scheme to ``wss``/``ws``.
+    """
+    url = settings.build_base_url(
+        model_name=model_name, api_config=_realtime_api_config(vendor_type)
+    )
+    suffix = f"/{REALTIME_API_TYPE}"
+    if url.endswith(suffix):
+        url = url[: -len(suffix)]
+    # Collapse accidental double slashes (e.g. a trailing slash in base_url),
+    # leaving the scheme's own "//" intact.
+    scheme, sep, rest = url.partition("://")
+    if sep:
+        url = f"{scheme}://{re.sub(r'/{2,}', '/', rest)}"
+    if url.startswith("https://"):
+        return "wss://" + url[len("https://") :]
+    if url.startswith("http://"):
+        return "ws://" + url[len("http://") :]
+    return url
+
+
+def _prepare_connection(
+    client: "OpenAI | AsyncOpenAI",
+    settings: UiPathBaseSettings,
+    *,
+    model: str,
+    vendor_type: str,
+    extra_headers: Headers,
+) -> Headers:
+    """Configure ``client`` for a gateway realtime connection to ``model``.
+
+    Sets ``websocket_base_url`` and the S2S bearer token (read straight from the
+    gateway auth handler), and returns the ``X-UiPath-*`` routing headers merged
+    with ``extra_headers`` for the WebSocket upgrade.
+    """
+    client.websocket_base_url = build_realtime_ws_base_url(
+        settings, model_name=model, vendor_type=vendor_type
+    )
+    auth = settings.build_auth_pipeline()
+    if isinstance(auth, LLMGatewayS2SAuth) and auth.access_token:
+        client.api_key = auth.access_token
+    merged: dict[str, object] = {
+        **settings.build_auth_headers(
+            model_name=model, api_config=_realtime_api_config(vendor_type)
+        )
+    }
+    if extra_headers:
+        merged.update(extra_headers)
+    return merged  # type: ignore[return-value]
+
+
+class _UiPathRealtime(Realtime):
+    """``Realtime`` resource that routes ``connect()`` through the gateway."""
+
+    def __init__(
+        self, client: OpenAI, *, settings: UiPathBaseSettings, model: str, vendor_type: str
+    ) -> None:
+        super().__init__(client)
+        self._uipath_settings = settings
+        self._uipath_model = model
+        self._uipath_vendor = vendor_type
+
+    @override
+    def connect(
+        self,
+        *,
+        call_id: str | Omit = omit,
+        model: str | Omit = omit,
+        extra_query: Query = {},
+        extra_headers: Headers = {},
+        websocket_connection_options: WebSocketConnectionOptions = {},
+    ) -> RealtimeConnectionManager:
+        resolved_model: str = self._uipath_model if model is omit else model  # type: ignore[assignment]
+        merged_headers = _prepare_connection(
+            self._client,
+            self._uipath_settings,
+            model=resolved_model,
+            vendor_type=self._uipath_vendor,
+            extra_headers=extra_headers,
+        )
+        return super().connect(
+            call_id=call_id,
+            model=resolved_model,
+            extra_query=extra_query,
+            extra_headers=merged_headers,
+            websocket_connection_options=websocket_connection_options,
+        )
+
+
+class _UiPathAsyncRealtime(AsyncRealtime):
+    """``AsyncRealtime`` resource that routes ``connect()`` through the gateway."""
+
+    def __init__(
+        self, client: AsyncOpenAI, *, settings: UiPathBaseSettings, model: str, vendor_type: str
+    ) -> None:
+        super().__init__(client)
+        self._uipath_settings = settings
+        self._uipath_model = model
+        self._uipath_vendor = vendor_type
+
+    @override
+    def connect(
+        self,
+        *,
+        call_id: str | Omit = omit,
+        model: str | Omit = omit,
+        extra_query: Query = {},
+        extra_headers: Headers = {},
+        websocket_connection_options: WebSocketConnectionOptions = {},
+    ) -> AsyncRealtimeConnectionManager:
+        resolved_model: str = self._uipath_model if model is omit else model  # type: ignore[assignment]
+        merged_headers = _prepare_connection(
+            self._client,
+            self._uipath_settings,
+            model=resolved_model,
+            vendor_type=self._uipath_vendor,
+            extra_headers=extra_headers,
+        )
+        return super().connect(
+            call_id=call_id,
+            model=resolved_model,
+            extra_query=extra_query,
+            extra_headers=merged_headers,
+            websocket_connection_options=websocket_connection_options,
+        )