diff --git a/.hooks/generate_docs.py b/.hooks/generate_docs.py index 1ab5b499..b38679be 100644 --- a/.hooks/generate_docs.py +++ b/.hooks/generate_docs.py @@ -96,7 +96,7 @@ def simple_convert_markdown( html_id: str = "", **kwargs: t.Any, ) -> t.Any: - return Markup(md.convert(text) if text else "") # noqa: S704 # nosec + return Markup(md.convert(text) if text else "") # nosec # noqa: S704 self.handler.env.filters["convert_markdown"] = simple_convert_markdown diff --git a/dreadnode/__init__.py b/dreadnode/__init__.py index 77de7bbb..0337148d 100644 --- a/dreadnode/__init__.py +++ b/dreadnode/__init__.py @@ -32,7 +32,7 @@ from dreadnode.version import VERSION if t.TYPE_CHECKING: - from dreadnode import scorers # noqa: F401 + from dreadnode import scorers logger.disable("dreadnode") diff --git a/dreadnode/agent/agent.py b/dreadnode/agent/agent.py index 22016af1..0a987c20 100644 --- a/dreadnode/agent/agent.py +++ b/dreadnode/agent/agent.py @@ -51,6 +51,8 @@ from dreadnode.agent.stop import StopCondition, stop_never from dreadnode.agent.thread import Thread from dreadnode.agent.tools import AnyTool, Tool, Toolset, discover_tools_on_obj +from dreadnode.agent.tools.planning import update_todo +from dreadnode.agent.tools.tasking import finish_task, give_up_on_task from dreadnode.agent.types import Message, ToolCall from dreadnode.meta import Component, Config, Model, component from dreadnode.scorers import ScorersLike @@ -745,9 +747,6 @@ class TaskAgent(Agent): """ def model_post_init(self, _: t.Any) -> None: - from dreadnode.agent.tools.planning import update_todo - from dreadnode.agent.tools.tasking import finish_task, give_up_on_task - if not any(tool for tool in self.tools if tool.name == "finish_task"): self.tools.append(finish_task) diff --git a/dreadnode/agent/console.py b/dreadnode/agent/console.py index 517a60c2..e73c01da 100644 --- a/dreadnode/agent/console.py +++ b/dreadnode/agent/console.py @@ -48,13 +48,13 @@ def _handle_tool_start(self, event: ToolStart) -> None: self._status_table, console=self.console, transient=True, auto_refresh=True ) self._live_status.start() + if self._status_table is not None: + # Add a new row for the running tool with a spinner. + self._status_table.add_row( + Text(f"Running [bold]{event.tool_call.name}[/bold]...", style="yellow") + ) - # Add a new row for the running tool with a spinner. - self._status_table.add_row( - Text(f"Running [bold]{event.tool_call.name}[/bold]...", style="yellow") - ) - - def _handle_tool_end(self, event: ToolEnd): + def _handle_tool_end(self, event: ToolEnd) -> None: """Prints the tool's result and cleans up the status board.""" # First, print the static result panel. This ensures it's in the # console history even after the live display is gone. diff --git a/dreadnode/agent/hooks/summarize.py b/dreadnode/agent/hooks/summarize.py index 0fac958a..82ccc080 100644 --- a/dreadnode/agent/hooks/summarize.py +++ b/dreadnode/agent/hooks/summarize.py @@ -1,6 +1,8 @@ import contextlib import typing as t +from litellm.exceptions import ContextWindowExceededError + from dreadnode.agent.events import AgentError, AgentEvent, GenerationEnd, StepStart from dreadnode.agent.prompts import summarize_conversation from dreadnode.agent.reactions import Continue, Reaction, Retry @@ -23,8 +25,6 @@ def _is_context_length_error(error: BaseException) -> bool: """Checks if an exception is likely due to exceeding the context window.""" with contextlib.suppress(ImportError): - from litellm.exceptions import ContextWindowExceededError - if isinstance(error, ContextWindowExceededError): return True diff --git a/dreadnode/agent/tools/fs.py b/dreadnode/agent/tools/fs.py index 7c31157f..7b76aac0 100644 --- a/dreadnode/agent/tools/fs.py +++ b/dreadnode/agent/tools/fs.py @@ -8,7 +8,7 @@ import rigging as rg from fsspec import AbstractFileSystem # type: ignore[import-untyped] from pydantic import PrivateAttr -from upath import UPath +from upath import UPath # type: ignore[import-not-found] from dreadnode.agent.tools import Toolset, tool_method from dreadnode.meta import Config @@ -127,7 +127,7 @@ def read_file( content = _path.read_bytes() try: - return content.decode("utf-8") + return content.decode("utf-8") # type: ignore[no-any-return] except UnicodeDecodeError as e: if self.multi_modal: return rg.ContentImageUrl.from_file(path) diff --git a/dreadnode/agent/tools/tasking.py b/dreadnode/agent/tools/tasking.py index 8af798fd..0f971d5a 100644 --- a/dreadnode/agent/tools/tasking.py +++ b/dreadnode/agent/tools/tasking.py @@ -5,7 +5,7 @@ @tool -async def finish_task(success: bool, summary: str) -> None: # noqa: ARG001, FBT001 +async def finish_task(success: bool, summary: str) -> None: # noqa: ARG001 """ Mark your task as complete with a success/failure status and markdown summary of actions taken. diff --git a/dreadnode/airt/attack/prompt.py b/dreadnode/airt/attack/prompt.py index 98b90f80..2b72941a 100644 --- a/dreadnode/airt/attack/prompt.py +++ b/dreadnode/airt/attack/prompt.py @@ -28,7 +28,7 @@ def prompt_attack( beam_width: int = 3, branching_factor: int = 3, max_steps: int = 10, - additional_scorers: list[Scorer] | None = None, + additional_scorers: list[Scorer[t.Any]] | None = None, name: str | None = None, ) -> Attack[str, str]: """ @@ -88,7 +88,7 @@ def prompt_attack( objective = weighted_avg( (judge_scorer, 1), - *[(scorer, 1) for scorer in additional_scorers], + *[(scorer, 1) for scorer in (additional_scorers or [])], name="prompt_objective", ) diff --git a/dreadnode/airt/attack/tap.py b/dreadnode/airt/attack/tap.py index 1fa06438..12aaa76c 100644 --- a/dreadnode/airt/attack/tap.py +++ b/dreadnode/airt/attack/tap.py @@ -18,7 +18,7 @@ def tap_attack( beam_width: int = 10, branching_factor: int = 3, max_steps: int = 10, - additional_constraints: list[Scorer] | None = None, + additional_constraints: list[Scorer[t.Any]] | None = None, ) -> Attack[str, str]: """ Creates a Generative Attack optimized for the TAP (Tree-of-thought Attack Prompting) pattern, diff --git a/dreadnode/airt/target/llm.py b/dreadnode/airt/target/llm.py index 33a0359c..963865db 100644 --- a/dreadnode/airt/target/llm.py +++ b/dreadnode/airt/target/llm.py @@ -35,6 +35,8 @@ class LLMTarget(Model, Target[t.Any, str]): @property def name(self) -> str: + if self._generator is None: + return "unknown" return self._generator.to_identifier(short=True) def as_task(self, input: t.Any) -> Task[[], str]: @@ -51,9 +53,12 @@ def as_task(self, input: t.Any) -> Task[[], str]: @task(name=f"generate - {self.name}", label="llm_target_generate", tags=["target"]) async def generate( - messages: list[rg.Message] = messages, params: rg.GenerateParams = params + messages: list[rg.Message] = messages, params: rg.GenerateParams | None = params ) -> str: - generated = (await self._generator.generate_messages([messages], [params]))[0] + if self._generator is None: + raise ValueError("Generator not initialized") + params_list = [params] if params is not None else [rg.GenerateParams()] + generated = (await self._generator.generate_messages([messages], params_list))[0] if isinstance(generated, BaseException): raise generated return generated.message.content diff --git a/dreadnode/artifact/credential_manager.py b/dreadnode/artifact/credential_manager.py index 242ce227..10116493 100644 --- a/dreadnode/artifact/credential_manager.py +++ b/dreadnode/artifact/credential_manager.py @@ -3,7 +3,7 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, TypeVar -from botocore.exceptions import ClientError +from botocore.exceptions import ClientError # type: ignore[import-untyped] from loguru import logger from s3fs import S3FileSystem # type: ignore[import-untyped] diff --git a/dreadnode/cli/agent/cli.py b/dreadnode/cli/agent/cli.py index f27c12b4..f2e37245 100644 --- a/dreadnode/cli/agent/cli.py +++ b/dreadnode/cli/agent/cli.py @@ -8,6 +8,7 @@ import rich from dreadnode import log_input +from dreadnode import run as run_span from dreadnode.agent import Agent from dreadnode.agent.format import format_agent, format_agents_table from dreadnode.discovery import DEFAULT_SEARCH_PATHS, discover @@ -118,8 +119,6 @@ async def agent_cli( *, config: t.Any = config_default, ) -> None: - from dreadnode import run as run_span - flat_config = {k: v for k, v in flatten_model(config).items() if v is not None} agent = hydrate(agent_blueprint, config) diff --git a/dreadnode/cli/main.py b/dreadnode/cli/main.py index 9ddca31c..3a4030f1 100644 --- a/dreadnode/cli/main.py +++ b/dreadnode/cli/main.py @@ -1,5 +1,7 @@ import contextlib +import importlib.metadata import pathlib +import platform import shutil import sys import typing as t @@ -202,10 +204,6 @@ def clone( @cli.command(help="Show versions and exit.", group="Meta") def version() -> None: - import importlib.metadata - import platform - import sys - version = importlib.metadata.version("dreadnode") python_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" diff --git a/dreadnode/cli/profile/cli.py b/dreadnode/cli/profile/cli.py index 45c2642f..9ed1ccf8 100644 --- a/dreadnode/cli/profile/cli.py +++ b/dreadnode/cli/profile/cli.py @@ -3,6 +3,7 @@ import cyclopts import rich from rich import box +from rich.prompt import Prompt from rich.table import Table from dreadnode.cli.api import Token @@ -59,8 +60,6 @@ def switch( # If no profile provided, prompt user to choose if profile is None: - from rich.prompt import Prompt - profiles = list(config.servers.keys()) rich.print("\nAvailable profiles:") for i, p in enumerate(profiles, 1): diff --git a/dreadnode/convert.py b/dreadnode/convert.py index c1fca0cd..ef762306 100644 --- a/dreadnode/convert.py +++ b/dreadnode/convert.py @@ -1,19 +1,17 @@ import typing as t -if t.TYPE_CHECKING: - import networkx as nx # type: ignore [import-untyped] +import networkx as nx # type: ignore[import-untyped] +if t.TYPE_CHECKING: from dreadnode.tracing.span import RunSpan def run_span_to_graph(run: "RunSpan") -> "nx.DiGraph": try: - import networkx as nx # pyright: ignore[reportMissingModuleSource] - except ImportError as e: + graph = nx.DiGraph() + except NameError as e: raise RuntimeError("The `networkx` package is required for graph conversion") from e - graph = nx.DiGraph() - graph.add_node( run.run_id, name=run.name, diff --git a/dreadnode/data_types/audio.py b/dreadnode/data_types/audio.py index 53ebf4e6..a992443d 100644 --- a/dreadnode/data_types/audio.py +++ b/dreadnode/data_types/audio.py @@ -1,27 +1,23 @@ +import importlib.util import io import typing as t from pathlib import Path -from dreadnode.data_types.base import DataType +import numpy as np -if t.TYPE_CHECKING: - import numpy as np +from dreadnode.data_types.base import DataType def check_imports() -> None: - try: - import soundfile as sf # type: ignore[import-untyped,unused-ignore] # noqa: F401 - except ImportError as e: + if importlib.util.find_spec("soundfile") is None: raise ImportError( "Audio processing requires SoundFile. Install with: pip install dreadnode[multimodal]" - ) from e + ) - try: - import numpy as np # type: ignore[import-untyped,unused-ignore] # noqa: F401 - except ImportError as e: + if importlib.util.find_spec("numpy") is None: raise ImportError( "Audio processing requires NumPy. Install with: pip install dreadnode[multimodal]" - ) from e + ) AudioDataType: t.TypeAlias = "str | Path | np.ndarray[t.Any, t.Any] | bytes" @@ -78,7 +74,6 @@ def _process_audio_data(self) -> tuple[bytes, str, int | None, float | None]: Returns: A tuple of (audio_bytes, format_name, sample_rate, duration) """ - import numpy as np if isinstance(self._data, str | Path) and Path(self._data).exists(): return self._process_file_path() @@ -94,7 +89,7 @@ def _process_file_path(self) -> tuple[bytes, str, int | None, float | None]: Returns: A tuple of (audio_bytes, format_name, sample_rate, duration) """ - import soundfile as sf # type: ignore[import-not-found,unused-ignore] + import soundfile as sf # type: ignore # noqa: PGH003 path_str = str(self._data) audio_bytes = Path(path_str).read_bytes() @@ -113,8 +108,7 @@ def _process_numpy_array(self) -> tuple[bytes, str, int | None, float | None]: Returns: A tuple of (audio_bytes, format_name, sample_rate, duration) """ - import numpy as np # type: ignore[import-not-found,unused-ignore] - import soundfile as sf # type: ignore[import-not-found,unused-ignore] + import soundfile as sf if self._sample_rate is None: raise ValueError('Argument "sample_rate" is required when using numpy arrays.') @@ -151,8 +145,6 @@ def _generate_metadata( Returns: A dictionary of metadata """ - import numpy as np # type: ignore[import-not-found,unused-ignore] - metadata: dict[str, str | int | float | None] = { "extension": format_name.lower(), "x-python-datatype": "dreadnode.Audio.bytes", diff --git a/dreadnode/data_types/base.py b/dreadnode/data_types/base.py index 9d566b16..ac16c4cd 100644 --- a/dreadnode/data_types/base.py +++ b/dreadnode/data_types/base.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import typing as t from abc import ABC, abstractmethod diff --git a/dreadnode/data_types/image.py b/dreadnode/data_types/image.py index 55ab8204..caa1ef12 100644 --- a/dreadnode/data_types/image.py +++ b/dreadnode/data_types/image.py @@ -3,27 +3,11 @@ import typing as t from pathlib import Path -from dreadnode.data_types.base import DataType - -if t.TYPE_CHECKING: - import numpy as np - - -def check_imports() -> None: - try: - import PIL # type: ignore[import,unused-ignore] # noqa: F401 - except ImportError as e: - raise ImportError( - "Image processing requires Pillow. Install with: pip install dreadnode[multimodal]" - ) from e - - try: - import numpy as np # type: ignore[import,unused-ignore] # noqa: F401 - except ImportError as e: - raise ImportError( - "Image processing requires NumPy. Install with: pip install dreadnode[multimodal]" - ) from e +import numpy as np +import PIL +import PIL.Image +from dreadnode.data_types.base import DataType ImageDataType = t.Union[t.Any, "np.ndarray[t.Any, t.Any]"] ImageDataOrPathType = str | Path | bytes | ImageDataType @@ -61,7 +45,6 @@ def __init__( caption: Optional caption for the image format: Optional format to use when saving (png, jpg, etc.) """ - check_imports() self._data = data self._mode = mode self._caption = caption @@ -83,10 +66,8 @@ def _process_image_data(self) -> tuple[bytes, str, str | None, int | None, int | Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import numpy as np # type: ignore[import,unused-ignore] - import PIL.Image # type: ignore[import,unused-ignore] - if isinstance(self._data, (str, Path)) and Path(self._data).exists(): + if isinstance(self._data, str | Path) and Path(self._data).exists(): return self._process_file_path() if isinstance(self._data, PIL.Image.Image): return self._process_pil_image() @@ -104,7 +85,6 @@ def _process_file_path(self) -> tuple[bytes, str, str | None, int | None, int | Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import PIL.Image # type: ignore[import,unused-ignore] path_str = str(self._data) image_bytes = Path(path_str).read_bytes() @@ -122,7 +102,6 @@ def _process_pil_image(self) -> tuple[bytes, str, str | None, int | None, int | Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import PIL.Image # type: ignore[import,unused-ignore] if not isinstance(self._data, PIL.Image.Image): raise TypeError(f"Expected PIL.Image, got {type(self._data)}") @@ -160,8 +139,6 @@ def _process_numpy_array(self) -> tuple[bytes, str, str | None, int | None, int Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import numpy as np # type: ignore[import,unused-ignore] - import PIL.Image # type: ignore[import,unused-ignore] buffer = io.BytesIO() image_format = self._format or "png" @@ -191,7 +168,6 @@ def _process_raw_bytes(self) -> tuple[bytes, str, str | None, int | None, int | Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import PIL.Image # type: ignore[import,unused-ignore] if not isinstance(self._data, bytes): raise TypeError(f"Expected bytes, got {type(self._data)}") @@ -216,7 +192,6 @@ def _process_base64_string(self) -> tuple[bytes, str, str | None, int | None, in Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import PIL.Image # type: ignore[import,unused-ignore] if not isinstance(self._data, str): raise TypeError(f"Expected str, got {type(self._data)}") @@ -253,15 +228,13 @@ def _generate_metadata( self, image_format: str, mode: str | None, width: int | None, height: int | None ) -> dict[str, str | int | None]: """Generate metadata for the image.""" - import numpy as np # type: ignore[import,unused-ignore] - import PIL.Image # type: ignore[import,unused-ignore] metadata: dict[str, str | int | None] = { "extension": image_format.lower(), "x-python-datatype": "dreadnode.Image.bytes", } - if isinstance(self._data, (str, Path)) and Path(self._data).exists(): + if isinstance(self._data, str | Path) and Path(self._data).exists(): metadata["source-type"] = "file" metadata["source-path"] = str(self._data) elif isinstance(self._data, PIL.Image.Image): @@ -313,7 +286,6 @@ def _ensure_valid_image_array( self, array: "np.ndarray[t.Any, np.dtype[t.Any]]" ) -> "np.ndarray[t.Any, np.dtype[t.Any]]": """Convert numpy array to a format suitable for PIL.""" - import numpy as np # type: ignore[import,unused-ignore] grayscale_dim = 2 rgb_dim = 3 diff --git a/dreadnode/data_types/object_3d.py b/dreadnode/data_types/object_3d.py index d4000983..bfaf745d 100644 --- a/dreadnode/data_types/object_3d.py +++ b/dreadnode/data_types/object_3d.py @@ -54,7 +54,7 @@ def to_serializable(self) -> tuple[bytes, dict[str, t.Any]]: Returns: A tuple of (object_bytes, metadata_dict) """ - if isinstance(self._data, (str, Path)) and Path(self._data).exists(): + if isinstance(self._data, str | Path) and Path(self._data).exists(): return self._process_file_path() if isinstance(self._data, bytes): format_name = self._format or "glb" @@ -67,7 +67,7 @@ def _process_file_path(self) -> tuple[bytes, dict[str, t.Any]]: Returns: A tuple of (object_bytes, metadata_dict) """ - if not isinstance(self._data, (str, Path)): + if not isinstance(self._data, str | Path): raise TypeError(f"Expected str or Path for file path, got {type(self._data)}") path = Path(self._data) object_bytes = path.read_bytes() @@ -92,7 +92,7 @@ def _generate_metadata(self, format_name: str) -> dict[str, t.Any]: if self._caption: metadata["caption"] = self._caption - if isinstance(self._data, (str, Path)): + if isinstance(self._data, str | Path): metadata["source-type"] = "file" metadata["source-path"] = str(self._data) elif isinstance(self._data, bytes): diff --git a/dreadnode/data_types/table.py b/dreadnode/data_types/table.py index 2446ff7f..46dbab58 100644 --- a/dreadnode/data_types/table.py +++ b/dreadnode/data_types/table.py @@ -3,11 +3,10 @@ from pathlib import Path from typing import ClassVar -from dreadnode.data_types.base import DataType +import numpy as np +import pandas as pd -if t.TYPE_CHECKING: - import numpy as np - import pandas as pd +from dreadnode.data_types.base import DataType TableDataType = t.Union[ "pd.DataFrame", dict[t.Any, t.Any], list[t.Any], str, Path, "np.ndarray[t.Any, t.Any]" @@ -78,12 +77,10 @@ def _to_dataframe(self) -> "pd.DataFrame": Returns: A pandas DataFrame representation of the input data """ - import numpy as np - import pandas as pd if isinstance(self._data, pd.DataFrame): return self._data - if isinstance(self._data, (str, Path)) and Path(self._data).exists(): + if isinstance(self._data, str | Path) and Path(self._data).exists(): path = Path(self._data) suffix = path.suffix.lower() @@ -98,7 +95,7 @@ def _to_dataframe(self) -> "pd.DataFrame": if isinstance(self._data, dict): return pd.DataFrame.from_dict(self._data) - if isinstance(self._data, (list, np.ndarray)): + if isinstance(self._data, list | np.ndarray): return pd.DataFrame(self._data) raise ValueError(f"Unsupported table data type: {type(self._data)}") @@ -134,8 +131,6 @@ def _generate_metadata(self, data_frame: "pd.DataFrame") -> dict[str, t.Any]: Returns: A dictionary of metadata """ - import numpy as np - import pandas as pd metadata = { "extension": self._format, @@ -151,7 +146,7 @@ def _generate_metadata(self, data_frame: "pd.DataFrame") -> dict[str, t.Any]: if isinstance(self._data, pd.DataFrame): metadata["source-type"] = "pandas.DataFrame" - elif isinstance(self._data, (str, Path)): + elif isinstance(self._data, str | Path): metadata["source-type"] = "file" metadata["source-path"] = str(self._data) elif isinstance(self._data, dict): diff --git a/dreadnode/data_types/video.py b/dreadnode/data_types/video.py index 54ef0f69..89e48925 100644 --- a/dreadnode/data_types/video.py +++ b/dreadnode/data_types/video.py @@ -71,11 +71,11 @@ def to_serializable(self) -> tuple[bytes, dict[str, t.Any]]: except ImportError: VideoClip = None # noqa: N806 - if isinstance(self._data, (str, Path)) and Path(self._data).exists(): + if isinstance(self._data, str | Path) and Path(self._data).exists(): return self._process_file_path() if isinstance(self._data, bytes): return self._process_bytes() - if isinstance(self._data, (np.ndarray, list)): + if isinstance(self._data, np.ndarray | list): return self._process_numpy_array() if VideoClip is not None and isinstance(self._data, VideoClip): return self._process_moviepy_clip() @@ -92,7 +92,7 @@ def _process_file_path(self) -> tuple[bytes, dict[str, t.Any]]: Returns: A tuple of (video_bytes, metadata_dict) """ - if not isinstance(self._data, (str, Path)): + if not isinstance(self._data, str | Path): raise TypeError("Expected file path as str or Path") video_bytes = Path(self._data).read_bytes() format_name = self._format @@ -126,7 +126,7 @@ def _process_numpy_array(self) -> tuple[bytes, dict[str, t.Any]]: if not self._fps: raise ValueError("fps is required for numpy array video frames") - if not isinstance(self._data, (np.ndarray, list)): + if not isinstance(self._data, np.ndarray | list): raise TypeError("data must be a numpy array or list of numpy arrays") frames = self._extract_frames_from_data() diff --git a/dreadnode/eval/console.py b/dreadnode/eval/console.py index f58dbe87..541b3931 100644 --- a/dreadnode/eval/console.py +++ b/dreadnode/eval/console.py @@ -19,7 +19,6 @@ from rich.table import Table from rich.text import Text -from dreadnode.eval.eval import In, Out from dreadnode.eval.events import ( EvalEnd, EvalEvent, @@ -38,7 +37,7 @@ EvalT = t.TypeVar("EvalT", bound="Eval") -class EvalConsoleAdapter(t.Generic[In, Out]): +class EvalConsoleAdapter: """ Consumes an Eval's event stream and renders a live progress dashboard. """ diff --git a/dreadnode/eval/dataset.py b/dreadnode/eval/dataset.py index 5c8db2ce..d5d4e515 100644 --- a/dreadnode/eval/dataset.py +++ b/dreadnode/eval/dataset.py @@ -3,6 +3,8 @@ import typing as t from pathlib import Path +import yaml # type: ignore[import-untyped] + from dreadnode.types import AnyDict FileFormat = t.Literal["jsonl", "csv", "json", "yaml", "yml"] @@ -49,13 +51,6 @@ def load_dataset(path: Path, *, file_format: FileFormat | None = None) -> list[A raise ValueError("JSON file must contain a list of objects.") elif file_format in {"yaml", "yml"}: - try: - import yaml # type: ignore[import-untyped,unused-ignore] - except ImportError as e: - raise ImportError( - "Loading YAML datasets requires PyYAML. Install with: pip install pyyaml" - ) from e - dataset = yaml.safe_load(content) if not isinstance(dataset, list): raise ValueError("YAML file must contain a list of objects.") diff --git a/dreadnode/eval/eval.py b/dreadnode/eval/eval.py index 7d0ae034..7e77cd14 100644 --- a/dreadnode/eval/eval.py +++ b/dreadnode/eval/eval.py @@ -9,6 +9,7 @@ from pydantic import ConfigDict, FilePath, TypeAdapter from dreadnode.discovery import find +from dreadnode.eval.console import EvalConsoleAdapter from dreadnode.eval.dataset import load_dataset from dreadnode.eval.events import ( EvalEnd, @@ -350,7 +351,6 @@ async def run(self) -> EvalResult[In, Out]: async def console(self) -> EvalResult: """Run the evaluation with a live display in the console.""" - from dreadnode.eval.console import EvalConsoleAdapter adapter = EvalConsoleAdapter(self) return await adapter.run() diff --git a/dreadnode/eval/result.py b/dreadnode/eval/result.py index 58a2dff2..bde39733 100644 --- a/dreadnode/eval/result.py +++ b/dreadnode/eval/result.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field from pathlib import Path +import pandas as pd import typing_extensions as te from dreadnode.eval.sample import Sample @@ -119,7 +120,6 @@ def to_dataframe(self) -> "t.Any": """ Converts the results into a pandas DataFrame for analysis. """ - import pandas as pd return pd.DataFrame(self.to_dicts()) # type: ignore[misc] @@ -129,8 +129,7 @@ def to_jsonl(self, path: str | Path) -> None: """ records = self.to_dicts() # type: ignore[misc] with Path(path).open("w", encoding="utf-8") as f: - for record in records: - f.write(json.dumps(record) + "\n") + f.writelines(json.dumps(record) + "\n" for record in records) @dataclass diff --git a/dreadnode/eval/sample.py b/dreadnode/eval/sample.py index a5d0bcf8..cd657c8f 100644 --- a/dreadnode/eval/sample.py +++ b/dreadnode/eval/sample.py @@ -88,7 +88,8 @@ def from_task( context: dict[str, t.Any] | None = None, ) -> "Sample[In, Out]": # Assume false for all - assertions = dict.fromkeys(task.assert_scores, False) + assert_scores: t.Any = getattr(task, "assert_scores", []) + assertions = dict.fromkeys(assert_scores, False) # If a score was reported, assume true for name in set(span.metrics.keys()) & set(assertions.keys()): diff --git a/dreadnode/meta/hydrate.py b/dreadnode/meta/hydrate.py index 83ff95de..a0873894 100644 --- a/dreadnode/meta/hydrate.py +++ b/dreadnode/meta/hydrate.py @@ -101,10 +101,12 @@ def _hydrate_recursive(obj: t.Any, override: t.Any) -> t.Any: # noqa: PLR0911, hydrated_dict[key] = _hydrate_recursive(item, item_overrides) return hydrated_dict - if not isinstance(obj, (str, int, float, bool, type(None), type)) and hasattr(obj, "__dict__"): + if not isinstance(obj, str | int | float | bool | type(None) | type) and hasattr( + obj, "__dict__" + ): with contextlib.suppress(Exception): for attr_name, attr_value in obj.__dict__.items(): - if attr_name.startswith("__") or not isinstance(attr_value, (Component, Model)): + if attr_name.startswith("__") or not isinstance(attr_value, Component | Model): continue hydrated = _hydrate_recursive(attr_value, override) diff --git a/dreadnode/meta/introspect.py b/dreadnode/meta/introspect.py index 8e36dd32..ce0a35db 100644 --- a/dreadnode/meta/introspect.py +++ b/dreadnode/meta/introspect.py @@ -152,10 +152,12 @@ def flatten_model(model: PydanticBaseModel, prefix: str = "") -> dict[str, t.Any def _find_nested_configurable(obj: t.Any) -> t.Any | None: - if isinstance(obj, (Component, Model)): + if isinstance(obj, Component | Model): return obj - if isinstance(obj, (str, int, float, bool, type(None), type)) or not hasattr(obj, "__dict__"): + if isinstance(obj, str | int | float | bool | type(None) | type) or not hasattr( + obj, "__dict__" + ): return None with contextlib.suppress(Exception): @@ -163,7 +165,7 @@ def _find_nested_configurable(obj: t.Any) -> t.Any | None: if attr_name.startswith("__"): continue - if isinstance(attr_value, (Component, Model)): + if isinstance(attr_value, Component | Model): return attr_value return None @@ -187,11 +189,11 @@ def _resolve_type_and_default(obj: t.Any, annotation: t.Any, name: str) -> tuple nested_fields: AnyDict = {} nested_default: t.Any - if isinstance(obj, (list, tuple)): + if isinstance(obj, list | tuple): used_names = set() for item in obj: - if not isinstance(item, (Model, Component)) and not ( + if not isinstance(item, Model | Component) and not ( item := _find_nested_configurable(item) ): continue @@ -221,7 +223,7 @@ def _resolve_type_and_default(obj: t.Any, annotation: t.Any, name: str) -> tuple elif isinstance(obj, dict): for key, value in obj.items(): - if not isinstance(value, (Model, Component)) and not ( + if not isinstance(value, Model | Component) and not ( value := _find_nested_configurable(value) ): continue @@ -238,7 +240,7 @@ def _resolve_type_and_default(obj: t.Any, annotation: t.Any, name: str) -> tuple with contextlib.suppress(Exception): obj_default = obj_type() - elif isinstance(obj, (Model, Component)): + elif isinstance(obj, Model | Component): obj_type = get_config_model(obj, name) obj_default = Ellipsis with contextlib.suppress(Exception): diff --git a/dreadnode/meta/types.py b/dreadnode/meta/types.py index 20876c37..18e361ae 100644 --- a/dreadnode/meta/types.py +++ b/dreadnode/meta/types.py @@ -279,7 +279,7 @@ def Config( # noqa: N802 ) class ConfigurableMeta(ModelMetaclass): def __new__( - mcs, + cls, name: str, bases: tuple[type[t.Any], ...], namespace: dict[str, t.Any], @@ -297,11 +297,11 @@ def __new__( } namespace[attr_name] = PydanticField(**field_kwargs) # type: ignore[arg-type] - cls = super().__new__(mcs, name, bases, namespace, **kwargs) + new_cls = super().__new__(cls, name, bases, namespace, **kwargs) # Merge config from all base classes merged_configs = {} - for base in reversed(cls.__mro__): # Go from most base to most derived + for base in reversed(new_cls.__mro__): # Go from most base to most derived if hasattr(base, "__dn_config__"): merged_configs.update(base.__dn_config__) @@ -309,13 +309,13 @@ def __new__( # If pydantic resolved any of our field descriptions, we need to # reflect those back into the ConfigInfo objects - for field_name, field_info in cls.model_fields.items(): # type: ignore[attr-defined] + for field_name, field_info in new_cls.model_fields.items(): # type: ignore[attr-defined] if field_name in configs: configs[field_name].field_kwargs["description"] = field_info.description - cls.__dn_config__ = merged_configs # type: ignore[attr-defined] + new_cls.__dn_config__ = merged_configs # type: ignore[attr-defined] - return cls + return new_cls class Model(PydanticBaseModel, metaclass=ConfigurableMeta): diff --git a/dreadnode/optimization/collectors.py b/dreadnode/optimization/collectors.py index 418a6ad9..ad8cb578 100644 --- a/dreadnode/optimization/collectors.py +++ b/dreadnode/optimization/collectors.py @@ -23,13 +23,13 @@ def get_parent(trial: Trial[T]) -> Trial[T] | None: else None ) - trials: Trials[T] = [] + trials: Trials[T] = [] # type: ignore[assignment] parent = get_parent(current_trial) while parent: trials.insert(0, parent) parent = get_parent(parent) - return trials[:depth] + return trials[:depth] # type: ignore[return-value] @component @@ -37,7 +37,7 @@ def all_successful(_: Trial[T], all_trials: Trials[T]) -> Trials[T]: """ Collects all successful trials, regardless of lineage. """ - return [t for t in all_trials if t.status == "success"] + return [t for t in all_trials if t.status == "success"] # type: ignore[return-value] @component @@ -53,14 +53,15 @@ def local_neighborhood( The maximum distance for any discovered node is `2h-1`. """ if not all_trials: - return [] + return [] # type: ignore[return-value] # 1 - Build a bi-directional graph for efficient traversal all_trials_map: dict[UUID, Trial] = {t.id: t for t in all_trials} children_map: dict[UUID, list[UUID]] = {tid: [] for tid in all_trials_map} for trial in [t_ for t_ in all_trials if t_.parent_id]: - children_map.setdefault(trial.parent_id, []).append(trial.id) + if trial.parent_id: + children_map.setdefault(trial.parent_id, []).append(trial.id) # 2 - Perform a BFS staying within 2h-1 @@ -92,4 +93,4 @@ def local_neighborhood( visited.add(child_id) queue.append((child_id, distance + 1)) - return [all_trials_map[tid] for tid in neighborhood_ids] + return [all_trials_map[tid] for tid in neighborhood_ids] # type: ignore[return-value] diff --git a/dreadnode/optimization/console.py b/dreadnode/optimization/console.py index 0fcde8be..d8342d33 100644 --- a/dreadnode/optimization/console.py +++ b/dreadnode/optimization/console.py @@ -33,7 +33,7 @@ from dreadnode.optimization.study import Study from dreadnode.optimization.trial import Trial -StudyT = t.TypeVar("StudyT", bound="Study") +StudyT = t.TypeVar("StudyT", bound="Study[t.Any]") class StudyConsoleAdapter(t.Generic[StudyT]): @@ -61,7 +61,7 @@ def __init__( expand=True, ) self._trial_log: deque[Trial] = deque(maxlen=max_log_entries) - self._best_trial_summary = Text("No successful trials yet.", style="dim") + self._best_trial_summary: Text | Table = Text("No successful trials yet.", style="dim") self._summary_stats = { "Successful": 0, "Failed": 0, @@ -143,7 +143,8 @@ def _build_dashboard(self) -> Panel: Layout(trials_panel, name="trials_log", ratio=1, minimum_size=5), ) - name = self.study.name or get_callable_name(self.study.objective, short=True) + objective_func = t.cast("t.Callable[..., t.Any]", self.study.objective) + name = self.study.name or get_callable_name(objective_func, short=True) return Panel( layout, @@ -151,7 +152,7 @@ def _build_dashboard(self) -> Panel: border_style="cyan", ) - def _handle_event(self, event: StudyEvent) -> None: # noqa: PLR0912 + def _handle_event(self, event: StudyEvent[t.Any]) -> None: # noqa: PLR0912 if isinstance(event, StudyStart): self._steps_task_id = self._progress.add_task( "[bold]Overall Steps", total=self.study.max_steps @@ -206,7 +207,7 @@ def _handle_event(self, event: StudyEvent) -> None: # noqa: PLR0912 if self._steps_task_id is not None: self._progress.update(self._steps_task_id, advance=1) if self._patience_task_id is not None: - if self.study._steps_since_best > 0: + if self.study._steps_since_best > 0: # noqa: SLF001 self._progress.update(self._patience_task_id, advance=1) else: self._progress.reset(self._patience_task_id) diff --git a/dreadnode/optimization/sampling.py b/dreadnode/optimization/sampling.py index a725e178..acc670ff 100644 --- a/dreadnode/optimization/sampling.py +++ b/dreadnode/optimization/sampling.py @@ -17,7 +17,7 @@ def top_k( Selects the top k trials by score (highest first). """ sorted_trials = sorted(trials, key=lambda t: t.score, reverse=True) - return sorted_trials[:k] + return sorted_trials[:k] # type: ignore[return-value] @component @@ -27,7 +27,7 @@ def random_( """ Selects k random trials from the pool. """ - return random.sample(trials, min(k, len(trials))) if trials else [] # nosec + return random.sample(trials, min(k, len(trials))) if trials else [] # type: ignore[return-value] @component @@ -47,9 +47,9 @@ def epsilon_greedy( if random.random() < epsilon and len(sorted_trials) >= k: # noqa: S311 # nosec k_minus_1 = sorted_trials[: k - 1] random_choice = random.choice(sorted_trials[k - 1 :]) # noqa: S311 # nosec - return [*k_minus_1, random_choice] + return [*k_minus_1, random_choice] # type: ignore[return-value] - return sorted_trials[:k] + return sorted_trials[:k] # type: ignore[return-value] @component @@ -72,7 +72,7 @@ def competitive(trials: Trials[T], *, k: int = Config(5), pool_size: int = Confi winners.append(winner) pool.remove(winner) - return winners + return winners # type: ignore[return-value] @component @@ -93,7 +93,7 @@ def proportional( A list of selected trials. """ if not trials: - return [] + return [] # type: ignore[return-value] # 1 - Normalize scores for use as weights @@ -104,7 +104,7 @@ def proportional( # If all trials have the same score - take the fast route if total_weight == 0: - return random.sample(trials, min(k, len(trials))) # nosec + return random.sample(trials, min(k, len(trials))) # type: ignore[return-value] # 2 - Select k winners one by one, without replacement @@ -128,7 +128,7 @@ def proportional( if sum(current_weights) == 0: break - return winners + return winners # type: ignore[return-value] # Utils @@ -144,7 +144,7 @@ def interleave_by_parent(trials: Trials[T]) -> Trials[T]: Example: `[P1, P1, P2, P2, P3]` -> [P1, P2, P3, P1, P2] """ if not trials: - return [] + return [] # type: ignore[return-value] parent_to_children = defaultdict(list) for trial in trials: @@ -156,4 +156,4 @@ def interleave_by_parent(trials: Trials[T]) -> Trials[T]: if trial is not None: interleaved_list.append(trial) # noqa: PERF401 - return interleaved_list + return interleaved_list # type: ignore[return-value] diff --git a/dreadnode/optimization/search/graph.py b/dreadnode/optimization/search/graph.py index fb00279e..f909159e 100644 --- a/dreadnode/optimization/search/graph.py +++ b/dreadnode/optimization/search/graph.py @@ -50,8 +50,8 @@ class GraphSearch(Model, Search[T]): pruning_sampler: TrialSampler[T] = Config(top_k) """A trial sampler to prune new children after each branching.""" - _trials: Trials[T] = PrivateAttr(default_factory=list) - _leaves: Trials[T] = PrivateAttr(default_factory=list) + _trials: Trials[T] = PrivateAttr(default_factory=list) # type: ignore[arg-type] + _leaves: Trials[T] = PrivateAttr(default_factory=list) # type: ignore[arg-type] def __repr__(self) -> str: parts = [ @@ -63,17 +63,17 @@ def __repr__(self) -> str: return f"GraphSearch({', '.join(parts)})" def reset(self) -> None: - self._trials = [] - self._leaves = [] + self._trials = [] # type: ignore[assignment] + self._leaves = [] # type: ignore[assignment] async def suggest(self, step: int) -> Trials[T]: if not self._leaves: - return [Trial(candidate=self.initial_candidate, step=step)] + return [Trial(candidate=self.initial_candidate, step=step)] # type: ignore[return-value] - trials: Trials[T] = [] + trials: Trials[T] = [] # type: ignore[assignment] for leaf in self._leaves: context = [leaf, *self.context_collector(leaf, self._trials)] - coroutines = [self.transform(context) for _ in range(self.branching_factor)] + coroutines = [self.transform(context) for _ in range(self.branching_factor)] # type: ignore[arg-type] for candidate in await asyncio.gather(*coroutines): trials.append(Trial(candidate=candidate, parent_id=leaf.id, step=step)) @@ -82,8 +82,8 @@ async def suggest(self, step: int) -> Trials[T]: async def observe(self, trials: list[Trial[T]]) -> None: successful_trials = [t for t in trials if t.status == "success"] self._trials.extend(successful_trials) - interleaved_trials = interleave_by_parent(successful_trials) # Prevent parent bias - self._leaves = self.pruning_sampler(interleaved_trials) + interleaved_trials: list[Trial[T]] = interleave_by_parent(successful_trials) # type: ignore[arg-type] + self._leaves = self.pruning_sampler(interleaved_trials) # type: ignore[arg-type] def iterative_search( @@ -109,11 +109,11 @@ def iterative_search( A pre-configured GraphSearch instance. """ return GraphSearch[T]( - transform=transform, + transform=transform, # type: ignore[arg-type] initial_candidate=initial_candidate, branching_factor=branching_factor, - context_collector=lineage(), - pruning_sampler=top_k(k=1), + context_collector=lineage(), # type: ignore[arg-type,call-arg] + pruning_sampler=top_k(k=1), # type: ignore[arg-type,call-arg] ) @@ -141,7 +141,7 @@ def beam_search( A pre-configured GraphSearch instance. """ return GraphSearch[T]( - transform=transform, + transform=transform, # type: ignore[arg-type] initial_candidate=initial_candidate, branching_factor=branching_factor, context_collector=lineage, @@ -176,7 +176,7 @@ def graph_search( A pre-configured GraphSearch instance. """ return GraphSearch[T]( - transform=transform, + transform=transform, # type: ignore[arg-type] initial_candidate=initial_candidate, branching_factor=branching_factor, context_collector=local_neighborhood.configure(depth=neighborhood_depth), diff --git a/dreadnode/optimization/search/optuna_.py b/dreadnode/optimization/search/optuna_.py index d4862e29..1dc2e214 100644 --- a/dreadnode/optimization/search/optuna_.py +++ b/dreadnode/optimization/search/optuna_.py @@ -42,7 +42,7 @@ def __init__( self.optuna_search_space = _convert_search_space(search_space) self._trial_map: dict[UUID, optuna.trial.Trial] = {} - async def suggest(self) -> list[Trial[AnyDict]]: + async def suggest(self, step: int) -> list[Trial[AnyDict]]: # type: ignore[override] # noqa: ARG002 optuna_trial = self.optuna_study.ask(self.optuna_search_space) candidate_params = optuna_trial.params @@ -53,7 +53,7 @@ async def suggest(self) -> list[Trial[AnyDict]]: return [trial] - def observe(self, trials: list[Trial[AnyDict]]) -> None: + async def observe(self, trials: list[Trial[AnyDict]]) -> None: for trial in trials: optuna_trial = self._trial_map[trial.id] if trial.status == "success": diff --git a/dreadnode/optimization/study.py b/dreadnode/optimization/study.py index d21d88b6..81cf2113 100644 --- a/dreadnode/optimization/study.py +++ b/dreadnode/optimization/study.py @@ -9,6 +9,7 @@ from dreadnode.eval.sample import InputDataset from dreadnode.meta import Model from dreadnode.meta.types import Config +from dreadnode.optimization.console import StudyConsoleAdapter from dreadnode.optimization.events import ( CandidatePruned, CandidatesSuggested, @@ -26,12 +27,10 @@ from dreadnode.optimization.trial import Trial, Trials from dreadnode.scorers import Scorer, ScorerLike from dreadnode.task import Task +from dreadnode.tracing.span import current_run_span from dreadnode.types import AnyDict from dreadnode.util import concurrent_gen, get_callable_name -if t.TYPE_CHECKING: - from dreadnode.eval.events import EvalStopReason - Direction = t.Literal["maximize", "minimize"] """The direction of optimization for the objective score.""" @@ -52,7 +51,7 @@ class Study(Model, t.Generic[CandidateT]): """The search strategy to use for suggesting new trials.""" task_factory: t.Callable[[CandidateT], Task[..., t.Any]] """A function that accepts a candidate and returns a configured Task ready for evaluation.""" - objective: ScorerLike[t.Any] | str = Config(default_factory=list) + objective: ScorerLike[t.Any] | str = Config(default_factory=list) # type: ignore[assignment] """The objective to optimize. Can be a scorer instance, a scorer-like callable, or a string name of scorer already on the task.""" dataset: InputDataset[t.Any] | list[AnyDict] | FilePath | None = Config( default=None, expose_as=FilePath | None @@ -142,8 +141,8 @@ async def _stream(self) -> t.AsyncGenerator[StudyEvent[CandidateT], None]: # no self._steps_since_best = 0 self.strategy.reset() - stop_reason: EvalStopReason = "unknown" - all_trials: Trials[CandidateT] = [] + stop_reason: t.Any = "unknown" + all_trials: Trials[CandidateT] = [] # type: ignore[assignment] best_trial: Trial[CandidateT] | None = None yield StudyStart(study=self, trials=all_trials, max_steps=self.max_steps) @@ -232,7 +231,7 @@ async def _stream(self) -> t.AsyncGenerator[StudyEvent[CandidateT], None]: # no result=StudyResult(trials=all_trials, stop_reason=stop_reason), ) - def _log_event_metrics(self, event: StudyEvent) -> None: + def _log_event_metrics(self, event: StudyEvent[t.Any]) -> None: from dreadnode import log_metric if isinstance(event, TrialComplete): @@ -249,7 +248,6 @@ def _log_event_metrics(self, event: StudyEvent) -> None: async def _stream_traced(self) -> t.AsyncGenerator[StudyEvent[CandidateT], None]: from dreadnode import log_inputs, log_outputs, log_params, run, task_span - from dreadnode.tracing.span import current_run_span objective_name = ( self.objective @@ -267,7 +265,7 @@ async def _stream_traced(self) -> t.AsyncGenerator[StudyEvent[CandidateT], None] # config_model = get_config_model(self) # flat_config = {k: v for k, v in flatten_model(config_model()).items() if v is not None} - flat_config = {} + flat_config: dict[str, t.Any] = {} with trace_context: if run_using_tasks: @@ -275,7 +273,7 @@ async def _stream_traced(self) -> t.AsyncGenerator[StudyEvent[CandidateT], None] else: log_params(**flat_config) - last_event: StudyEvent | None = None + last_event: StudyEvent[t.Any] | None = None try: async with contextlib.aclosing(self._stream()) as stream: async for event in stream: @@ -287,10 +285,10 @@ async def _stream_traced(self) -> t.AsyncGenerator[StudyEvent[CandidateT], None] result = last_event.result outputs = {"stop_reason": result.stop_reason} if result.best_trial: - outputs["best_score"] = result.best_trial.score + outputs["best_score"] = result.best_trial.score # type: ignore[assignment] outputs["best_candidate"] = result.best_trial.candidate - outputs["best_output"] = result.best_trial.output - log_outputs(**outputs) + outputs["best_output"] = result.best_trial.output # type: ignore[assignment] + log_outputs(**outputs) # type: ignore[arg-type] @contextlib.asynccontextmanager async def stream(self) -> t.AsyncIterator[t.AsyncGenerator[StudyEvent[CandidateT], None]]: @@ -335,7 +333,6 @@ async def run(self) -> StudyResult[CandidateT]: async def console(self) -> StudyResult[CandidateT]: """Runs the optimization study with a live progress dashboard in the console.""" - from dreadnode.optimization.console import StudyConsoleAdapter adapter = StudyConsoleAdapter(self) return await adapter.run() diff --git a/dreadnode/optimization/trial.py b/dreadnode/optimization/trial.py index 5ca4d36a..f00720df 100644 --- a/dreadnode/optimization/trial.py +++ b/dreadnode/optimization/trial.py @@ -39,7 +39,6 @@ def __repr__(self) -> str: return f"Trial(id={self.id}, status='{self.status}', score={self.score:.3f})" @computed_field - @property def output(self) -> t.Any | None: """Get the output of the trial.""" if self.eval_result and self.eval_result.samples: diff --git a/dreadnode/scorers/classification.py b/dreadnode/scorers/classification.py index a96a0162..4c2b6577 100644 --- a/dreadnode/scorers/classification.py +++ b/dreadnode/scorers/classification.py @@ -1,5 +1,7 @@ import typing as t +from transformers import pipeline + from dreadnode.meta import Config from dreadnode.metric import Metric from dreadnode.scorers import Scorer @@ -35,9 +37,7 @@ def zero_shot_classification( ) try: - from transformers import ( # type: ignore [attr-defined,import-not-found,unused-ignore] - pipeline, - ) + pipeline("zero-shot-classification", model=model_name) except ImportError: warn_at_user_stacklevel(transformers_error_msg, UserWarning) @@ -45,6 +45,12 @@ def disabled_evaluate(_: t.Any) -> Metric: return Metric(value=0.0, attributes={"error": transformers_error_msg}) return Scorer(disabled_evaluate, name=name) + except Exception: # noqa: BLE001 + + def disabled_evaluate(_: t.Any) -> Metric: + return Metric(value=0.0, attributes={"error": "Failed to create pipeline"}) + + return Scorer(disabled_evaluate, name=name) def evaluate( data: t.Any, diff --git a/dreadnode/scorers/harm.py b/dreadnode/scorers/harm.py index 80e66b76..2ac52821 100644 --- a/dreadnode/scorers/harm.py +++ b/dreadnode/scorers/harm.py @@ -1,15 +1,11 @@ import typing as t +import openai + from dreadnode.meta import Config from dreadnode.metric import Metric from dreadnode.scorers import Scorer -if t.TYPE_CHECKING: - import openai - -if t.TYPE_CHECKING: - import openai - def detect_harm_with_openai( *, @@ -36,7 +32,6 @@ def detect_harm_with_openai( model: The moderation model to use. name: Name of the scorer. """ - import openai async def evaluate( data: t.Any, *, api_key: str | None = Config(api_key), model: str = Config(model) diff --git a/dreadnode/scorers/pii.py b/dreadnode/scorers/pii.py index c96a2131..ba1c9c35 100644 --- a/dreadnode/scorers/pii.py +++ b/dreadnode/scorers/pii.py @@ -1,14 +1,15 @@ import re import typing as t +from presidio_analyzer import AnalyzerEngine +from presidio_analyzer.nlp_engine import NlpEngineProvider + from dreadnode.metric import Metric from dreadnode.scorers import Scorer from dreadnode.scorers.contains import contains from dreadnode.util import warn_at_user_stacklevel if t.TYPE_CHECKING: - from presidio_analyzer import AnalyzerEngine # type: ignore[import-not-found,unused-ignore] - from dreadnode.types import JsonDict @@ -65,9 +66,6 @@ def _get_presidio_analyzer() -> "AnalyzerEngine": """Lazily initializes and returns a singleton Presidio AnalyzerEngine instance.""" global g_analyzer_engine # noqa: PLW0603 - from presidio_analyzer import AnalyzerEngine - from presidio_analyzer.nlp_engine import NlpEngineProvider - if g_analyzer_engine is None: provider = NlpEngineProvider( nlp_configuration={ @@ -109,8 +107,8 @@ def detect_pii_with_presidio( ) try: - import presidio_analyzer # type: ignore[import-not-found,unused-ignore] # noqa: F401 - except ImportError: + _get_presidio_analyzer() + except (ImportError, OSError): warn_at_user_stacklevel(presidio_import_error_msg, UserWarning) def disabled_evaluate(_: t.Any) -> Metric: diff --git a/dreadnode/scorers/readability.py b/dreadnode/scorers/readability.py index e8fa3373..259f5feb 100644 --- a/dreadnode/scorers/readability.py +++ b/dreadnode/scorers/readability.py @@ -1,5 +1,7 @@ import typing as t +import textstat # type: ignore[import-untyped] + from dreadnode.metric import Metric from dreadnode.scorers.base import Scorer from dreadnode.util import warn_at_user_stacklevel @@ -27,8 +29,8 @@ def readability( ) try: - import textstat # type: ignore[import-not-found,import-untyped,unused-ignore] - except ImportError: + textstat.flesch_kincaid_grade("test") + except (ImportError, AttributeError): warn_at_user_stacklevel(textstat_import_error_msg, UserWarning) def disabled_evaluate(_: t.Any) -> Metric: diff --git a/dreadnode/scorers/rigging.py b/dreadnode/scorers/rigging.py index 48f76e03..9d3a3bd6 100644 --- a/dreadnode/scorers/rigging.py +++ b/dreadnode/scorers/rigging.py @@ -1,10 +1,11 @@ import typing as t +from rigging.chat import Chat + from dreadnode.metric import Metric from dreadnode.scorers.base import Scorer if t.TYPE_CHECKING: - from rigging.chat import Chat from rigging.message import Message ChatFilterMode = t.Literal[ @@ -44,8 +45,6 @@ def wrap_chat( """ async def evaluate(chat: "Chat") -> Metric: - from rigging.chat import Chat - # Fall through to the inner scorer if chat is not a Chat instance if not isinstance(chat, Chat): return await inner_scorer(chat) diff --git a/dreadnode/scorers/sentiment.py b/dreadnode/scorers/sentiment.py index 895d4b36..d9151da8 100644 --- a/dreadnode/scorers/sentiment.py +++ b/dreadnode/scorers/sentiment.py @@ -2,6 +2,7 @@ import typing as t import httpx +from textblob import TextBlob # type: ignore[import-untyped] from dreadnode.meta import Config from dreadnode.metric import Metric @@ -32,8 +33,8 @@ def sentiment( textblob_import_error_msg = "TextBlob dependency is not installed. Install with: pip install textblob && python -m textblob.download_corpora" try: - from textblob import TextBlob # type: ignore[import-not-found,unused-ignore,import-untyped] - except ImportError: + TextBlob("test").sentiment # noqa: B018 + except (ImportError, AttributeError): warn_at_user_stacklevel(textblob_import_error_msg, UserWarning) def disabled_evaluate(_: t.Any) -> Metric: diff --git a/dreadnode/scorers/similarity.py b/dreadnode/scorers/similarity.py index 0bd2db80..de199751 100644 --- a/dreadnode/scorers/similarity.py +++ b/dreadnode/scorers/similarity.py @@ -1,17 +1,23 @@ import typing as t from difflib import SequenceMatcher +import litellm +import nltk # type: ignore[import-untyped] +from nltk.tokenize import word_tokenize # type: ignore[import-untyped] +from nltk.translate.bleu_score import sentence_bleu # type: ignore[import-untyped] +from rapidfuzz import distance, fuzz, utils +from sentence_transformers import SentenceTransformer, util +from sklearn.feature_extraction.text import TfidfVectorizer # type: ignore[import-untyped] +from sklearn.metrics.pairwise import ( # type: ignore # noqa: PGH003 + cosine_similarity as sklearn_cosine_similarity, +) + from dreadnode.meta import Config from dreadnode.metric import Metric from dreadnode.scorers.base import Scorer from dreadnode.scorers.util import cosine_similarity from dreadnode.util import warn_at_user_stacklevel -if t.TYPE_CHECKING: - from sentence_transformers import ( # type: ignore[import-not-found,import-untyped,unused-ignore] - SentenceTransformer, - ) - def similarity( reference: str, @@ -93,7 +99,7 @@ def similarity_with_rapidfuzz( ) try: - from rapidfuzz import fuzz, utils # type: ignore[import-not-found,unused-ignore] + fuzz.ratio("test", "test") except ImportError: warn_at_user_stacklevel(rapidfuzz_import_error_msg, UserWarning) @@ -162,7 +168,7 @@ def evaluate( return Scorer(evaluate, name=name, catch=True) -def distance( +def string_distance( reference: str, *, method: t.Literal[ @@ -190,7 +196,7 @@ def distance( ) try: - from rapidfuzz import distance # type: ignore[import-not-found,unused-ignore] + distance.Levenshtein.distance("test", "test") except ImportError: warn_at_user_stacklevel(rapidfuzz_import_error_msg, UserWarning) @@ -259,12 +265,7 @@ def similarity_with_tf_idf(reference: str, *, name: str = "similarity") -> "Scor ) try: - from sklearn.feature_extraction.text import ( # type: ignore[import-not-found,unused-ignore] - TfidfVectorizer, - ) - from sklearn.metrics.pairwise import ( # type: ignore[import-not-found,unused-ignore] - cosine_similarity as sklearn_cosine_similarity, - ) + TfidfVectorizer() except ImportError: warn_at_user_stacklevel(sklearn_import_error_msg, UserWarning) @@ -311,10 +312,7 @@ def similarity_with_sentence_transformers( sentence_transformers_error_msg = "Sentence transformers dependency is not installed. Please install it with: pip install sentence-transformers" try: - from sentence_transformers import ( # type: ignore[import-not-found,import-untyped,unused-ignore] - SentenceTransformer, - util, - ) + SentenceTransformer(model_name) except ImportError: warn_at_user_stacklevel(sentence_transformers_error_msg, UserWarning) @@ -372,7 +370,6 @@ def similarity_with_litellm( or self-hosted models. name: Name of the scorer. """ - import litellm async def evaluate( data: t.Any, @@ -429,21 +426,17 @@ def bleu( nltk_import_error_msg = "NLTK dependency is not installed. Install with: pip install nltk && python -m nltk.downloader punkt" try: - import nltk # type: ignore[import-not-found,unused-ignore] - from nltk.tokenize import word_tokenize # type: ignore[import-not-found,unused-ignore] - from nltk.translate.bleu_score import ( # type: ignore[import-not-found,unused-ignore] - sentence_bleu, - ) - # Check for the 'punkt' tokenizer data try: nltk.data.find("tokenizers/punkt") + word_tokenize("test") + sentence_bleu([["test"]], ["test"]) except LookupError as e: nltk_import_error_msg = ( "NLTK 'punkt' tokenizer not found. Please run: python -m nltk.downloader punkt" ) raise ImportError(nltk_import_error_msg) from e - except ImportError: + except (ImportError, AttributeError): warn_at_user_stacklevel(nltk_import_error_msg, UserWarning) def disabled_evaluate(_: t.Any) -> Metric: diff --git a/dreadnode/serialization.py b/dreadnode/serialization.py index 462acf0e..834ce5b7 100644 --- a/dreadnode/serialization.py +++ b/dreadnode/serialization.py @@ -23,6 +23,14 @@ from re import Pattern from uuid import UUID +import attrs +import datasets # type: ignore[import-untyped] +import numpy as np +import pandas as pd +import pydantic +import pydantic.dataclasses +from pydantic import TypeAdapter + from dreadnode.data_types.base import DataType from dreadnode.types import JsonDict, JsonValue from dreadnode.util import safe_repr @@ -317,16 +325,11 @@ def _handle_dataclass(obj: t.Any, seen: set[int]) -> tuple[JsonValue, JsonDict]: def _handle_attrs(obj: t.Any, seen: set[int]) -> tuple[JsonValue, JsonDict]: - import attrs - keys = [f.name for f in attrs.fields(obj.__class__)] return _handle_custom_object(obj, keys, seen, "attrs") def _handle_pydantic_dataclass(obj: t.Any, _seen: set[int]) -> tuple[JsonValue, JsonDict]: - import pydantic.dataclasses - from pydantic import TypeAdapter - if not pydantic.dataclasses.is_pydantic_dataclass(obj.__class__): return safe_repr(obj), UNKNOWN_OBJECT_SCHEMA @@ -341,8 +344,6 @@ def _handle_pydantic_dataclass(obj: t.Any, _seen: set[int]) -> tuple[JsonValue, def _handle_pydantic_model(obj: t.Any, _seen: set[int]) -> tuple[JsonValue, JsonDict]: - import pydantic - if not isinstance(obj, pydantic.BaseModel): return safe_repr(obj), UNKNOWN_OBJECT_SCHEMA @@ -362,9 +363,7 @@ def _handle_numpy_array( obj: t.Any, seen: set[int], ) -> tuple[JsonValue, JsonDict]: - import numpy # noqa: ICN001 - - if not isinstance(obj, numpy.ndarray): + if not isinstance(obj, np.ndarray): return safe_repr(obj), UNKNOWN_OBJECT_SCHEMA serialized, schema = _handle_bytes(obj.tobytes(), seen) @@ -380,8 +379,6 @@ def _handle_pandas_dataframe( obj: t.Any, seen: set[int], ) -> tuple[JsonValue, JsonDict]: - import pandas as pd - if not isinstance(obj, pd.DataFrame): return safe_repr(obj), UNKNOWN_OBJECT_SCHEMA @@ -395,8 +392,6 @@ def _handle_pandas_series( obj: t.Any, seen: set[int], ) -> tuple[JsonValue, JsonDict]: - import pandas as pd - if not isinstance(obj, pd.Series): return safe_repr(obj), UNKNOWN_OBJECT_SCHEMA @@ -407,8 +402,6 @@ def _handle_pandas_series( def _handle_dataset(obj: t.Any, _seen: set[int]) -> tuple[JsonValue, JsonDict]: - import datasets # type: ignore[import-untyped] - if not isinstance(obj, datasets.Dataset): return safe_repr(obj), UNKNOWN_OBJECT_SCHEMA @@ -470,8 +463,6 @@ def _get_handlers() -> dict[type, HandlerFunc]: # Pydantic with contextlib.suppress(Exception): - import pydantic - handlers[pydantic.NameEmail] = lambda o, s: _handle_str_based( o, s, @@ -495,8 +486,6 @@ def _get_handlers() -> dict[type, HandlerFunc]: handlers[pydantic.BaseModel] = _handle_pydantic_model with contextlib.suppress(Exception): - import numpy as np - handlers[np.ndarray] = _handle_numpy_array handlers[np.floating] = lambda o, s: _serialize(float(o), s) handlers[np.integer] = lambda o, s: _serialize(int(o), s) @@ -513,14 +502,10 @@ def _get_handlers() -> dict[type, HandlerFunc]: ) with contextlib.suppress(Exception): - import pandas as pd - handlers[pd.DataFrame] = _handle_pandas_dataframe handlers[pd.Series] = _handle_pandas_series with contextlib.suppress(Exception): - import datasets - handlers[datasets.Dataset] = _handle_dataset with contextlib.suppress(Exception): @@ -572,8 +557,6 @@ def _serialize(obj: t.Any, seen: set[int] | None = None) -> tuple[JsonValue, Jso if dataclasses.is_dataclass(obj) and not isinstance(obj, type): with contextlib.suppress(Exception): - import pydantic.dataclasses - if pydantic.dataclasses.is_pydantic_dataclass(obj.__class__): return _handle_pydantic_dataclass(obj, seen) diff --git a/dreadnode/task.py b/dreadnode/task.py index 0e6cda0d..fe05285d 100644 --- a/dreadnode/task.py +++ b/dreadnode/task.py @@ -403,7 +403,7 @@ def as_target( self, input_param_name: str | None = None, ) -> "CustomTarget[R]": - from dreadnode.airt.target import CustomTarget + from dreadnode.airt.target.custom import CustomTarget return CustomTarget(task=self, input_param_name=input_param_name) diff --git a/dreadnode/transforms/ascii_art.py b/dreadnode/transforms/ascii_art.py index 7e0178ec..0c62af7e 100644 --- a/dreadnode/transforms/ascii_art.py +++ b/dreadnode/transforms/ascii_art.py @@ -1,3 +1,5 @@ +from art import text2art # type: ignore[import-untyped] + from dreadnode.meta import Config from dreadnode.transforms.base import Transform @@ -6,11 +8,11 @@ def ascii_art(font: str = "rand", *, name: str = "ascii_art") -> Transform[str, """Converts text into ASCII art using the 'art' library.""" try: - from art import text2art # type: ignore[import-not-found,unused-ignore,import-untyped] - except ImportError as e: + text2art("test") # Test if art is working + except (ImportError, AttributeError): raise ImportError( "ASCII art dependency is not installed. Install with: pip install art" - ) from e + ) from ImportError("art library not available") def transform(text: str, *, font: str = Config(font, help="The font to use")) -> str: return str(text2art(text, font=font)) diff --git a/dreadnode/transforms/llm_refine.py b/dreadnode/transforms/llm_refine.py index f3088463..b49f8d2f 100644 --- a/dreadnode/transforms/llm_refine.py +++ b/dreadnode/transforms/llm_refine.py @@ -4,10 +4,12 @@ import rigging as rg from dreadnode.meta import Config -from dreadnode.optimization.trial import Trials from dreadnode.transforms.base import Transform from dreadnode.types import AnyDict +if t.TYPE_CHECKING: + from dreadnode.optimization.trial import Trials + T = t.TypeVar("T") diff --git a/dreadnode/transforms/perturbation.py b/dreadnode/transforms/perturbation.py index 59d82bd5..cf9bf01f 100644 --- a/dreadnode/transforms/perturbation.py +++ b/dreadnode/transforms/perturbation.py @@ -3,6 +3,8 @@ import typing as t import unicodedata +from confusables import confusable_characters # type: ignore[import-untyped] + from dreadnode.meta import Config from dreadnode.transforms.base import Transform @@ -224,13 +226,11 @@ def unicode_confusable( """ try: - from confusables import ( # type: ignore[import-not-found,unused-ignore,import-untyped] - confusable_characters, - ) - except ImportError as e: + confusable_characters("a") + except (ImportError, AttributeError): raise ImportError( "Confusables dependency is not installed. Install with: pip install confusables" - ) from e + ) from ImportError("confusables library not available") if not 0.0 <= ratio <= 1.0: raise ValueError("Application ratio must be between 0.0 and 1.0.") diff --git a/poetry.lock b/poetry.lock index 273a34d8..0a331377 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "aiobotocore" @@ -285,7 +285,7 @@ description = "Timeout context manager for asyncio programs" optional = false python-versions = ">=3.8" groups = ["main"] -markers = "python_version < \"3.11\"" +markers = "python_version == \"3.10\"" files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, @@ -1623,7 +1623,7 @@ description = "IPython: Productive Interactive Computing" optional = false python-versions = ">=3.10" groups = ["main"] -markers = "python_version < \"3.11\"" +markers = "python_version == \"3.10\"" files = [ {file = "ipython-8.37.0-py3-none-any.whl", hash = "sha256:ed87326596b878932dbcb171e3e698845434d8c61b8d8cd474bf663041a9dcf2"}, {file = "ipython-8.37.0.tar.gz", hash = "sha256:ca815841e1a41a1e6b73a0b08f3038af9b2252564d01fc405356d34033012216"}, @@ -1872,7 +1872,7 @@ description = "Lightweight pipelining with Python functions" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"text\"" +markers = "extra == \"training\" or extra == \"text\"" files = [ {file = "joblib-1.5.2-py3-none-any.whl", hash = "sha256:4e1f0bdbb987e6d843c70cf43714cb276623def372df3c22fe5266b2670bc241"}, {file = "joblib-1.5.2.tar.gz", hash = "sha256:3faa5c39054b2f03ca547da9b2f52fde67c06240c31853f306aea97f13647b55"}, @@ -2568,6 +2568,25 @@ doc = ["Sphinx (==6.*)", "numpydoc (<2.0)", "pydata-sphinx-theme (==0.13)", "sph lint = ["black (>=23.7.0)", "flake8 (>=6.0.0)", "flake8-absolute-import (>=1.0)", "flake8-docstrings (>=1.7.0)", "flake8-implicit-str-concat (==0.4.0)", "flake8-rst-docstrings (>=0.3)", "isort (>=5.12)", "pre-commit (>=3.3)"] test = ["coveralls (>=3.0,<4.0)", "pytest (>=3.0.0,<7.0.0)", "pytest-cov (>=2.5.1,<3.0)"] +[[package]] +name = "mpmath" +version = "1.3.0" +description = "Python library for arbitrary-precision floating-point arithmetic" +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"training\"" +files = [ + {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, + {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, +] + +[package.extras] +develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] +docs = ["sphinx"] +gmpy = ["gmpy2 (>=2.1.0a4) ; platform_python_implementation != \"PyPy\""] +tests = ["pytest (>=4.6)"] + [[package]] name = "multidict" version = "6.6.4" @@ -2851,6 +2870,49 @@ files = [ {file = "nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe"}, ] +[[package]] +name = "networkx" +version = "3.4.2" +description = "Python package for creating and manipulating graphs and networks" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version == \"3.10\"" +files = [ + {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, + {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, +] + +[package.extras] +default = ["matplotlib (>=3.7)", "numpy (>=1.24)", "pandas (>=2.0)", "scipy (>=1.10,!=1.11.0,!=1.11.1)"] +developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] +doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.15)", "sphinx (>=7.3)", "sphinx-gallery (>=0.16)", "texext (>=0.6.7)"] +example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "momepy (>=0.7.2)", "osmnx (>=1.9)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] +extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] +test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] + +[[package]] +name = "networkx" +version = "3.5" +description = "Python package for creating and manipulating graphs and networks" +optional = false +python-versions = ">=3.11" +groups = ["main"] +markers = "python_version >= \"3.11\"" +files = [ + {file = "networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec"}, + {file = "networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037"}, +] + +[package.extras] +default = ["matplotlib (>=3.8)", "numpy (>=1.25)", "pandas (>=2.0)", "scipy (>=1.11.2)"] +developer = ["mypy (>=1.15)", "pre-commit (>=4.1)"] +doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=10)", "pydata-sphinx-theme (>=0.16)", "sphinx (>=8.0)", "sphinx-gallery (>=0.18)", "texext (>=0.6.7)"] +example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "momepy (>=0.7.2)", "osmnx (>=2.0.0)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] +extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] +test = ["pytest (>=7.2)", "pytest-cov (>=4.0)", "pytest-xdist (>=3.0)"] +test-extras = ["pytest-mpl", "pytest-randomly"] + [[package]] name = "nltk" version = "3.9.1" @@ -2898,7 +2960,7 @@ description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.10" groups = ["main"] -markers = "python_version < \"3.11\"" +markers = "python_version == \"3.10\"" files = [ {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, @@ -3042,6 +3104,214 @@ files = [ {file = "numpy-2.3.2.tar.gz", hash = "sha256:e0486a11ec30cdecb53f184d496d1c6a20786c81e55e41640270130056f8ee48"}, ] +[[package]] +name = "nvidia-cublas-cu12" +version = "12.8.4.1" +description = "CUBLAS native runtime libraries" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0"}, + {file = "nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142"}, + {file = "nvidia_cublas_cu12-12.8.4.1-py3-none-win_amd64.whl", hash = "sha256:47e9b82132fa8d2b4944e708049229601448aaad7e6f296f630f2d1a32de35af"}, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.8.90" +description = "CUDA profiling tools runtime libs." +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed"}, + {file = "nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182"}, + {file = "nvidia_cuda_cupti_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:bb479dcdf7e6d4f8b0b01b115260399bf34154a1a2e9fe11c85c517d87efd98e"}, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.8.93" +description = "NVRTC native runtime libraries" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994"}, + {file = "nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc1fec1e1637854b4c0a65fb9a8346b51dd9ee69e61ebaccc82058441f15bce8"}, + {file = "nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:7a4b6b2904850fe78e0bd179c4b655c404d4bb799ef03ddc60804247099ae909"}, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.8.90" +description = "CUDA Runtime native Libraries" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d"}, + {file = "nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90"}, + {file = "nvidia_cuda_runtime_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:c0c6027f01505bfed6c3b21ec546f69c687689aad5f1a377554bc6ca4aa993a8"}, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.10.2.21" +description = "cuDNN runtime libraries" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8"}, + {file = "nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8"}, + {file = "nvidia_cudnn_cu12-9.10.2.21-py3-none-win_amd64.whl", hash = "sha256:c6288de7d63e6cf62988f0923f96dc339cea362decb1bf5b3141883392a7d65e"}, +] + +[package.dependencies] +nvidia-cublas-cu12 = "*" + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.3.83" +description = "CUFFT native runtime libraries" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a"}, + {file = "nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74"}, + {file = "nvidia_cufft_cu12-11.3.3.83-py3-none-win_amd64.whl", hash = "sha256:7a64a98ef2a7c47f905aaf8931b69a3a43f27c55530c698bb2ed7c75c0b42cb7"}, +] + +[package.dependencies] +nvidia-nvjitlink-cu12 = "*" + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.1.3" +description = "cuFile GPUDirect libraries" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc"}, + {file = "nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:4beb6d4cce47c1a0f1013d72e02b0994730359e17801d395bdcbf20cfb3bb00a"}, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.9.90" +description = "CURAND native runtime libraries" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd"}, + {file = "nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9"}, + {file = "nvidia_curand_cu12-10.3.9.90-py3-none-win_amd64.whl", hash = "sha256:f149a8ca457277da854f89cf282d6ef43176861926c7ac85b2a0fbd237c587ec"}, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.3.90" +description = "CUDA solver native runtime libraries" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0"}, + {file = "nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450"}, + {file = "nvidia_cusolver_cu12-11.7.3.90-py3-none-win_amd64.whl", hash = "sha256:4a550db115fcabc4d495eb7d39ac8b58d4ab5d8e63274d3754df1c0ad6a22d34"}, +] + +[package.dependencies] +nvidia-cublas-cu12 = "*" +nvidia-cusparse-cu12 = "*" +nvidia-nvjitlink-cu12 = "*" + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.8.93" +description = "CUSPARSE native runtime libraries" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc"}, + {file = "nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b"}, + {file = "nvidia_cusparse_cu12-12.5.8.93-py3-none-win_amd64.whl", hash = "sha256:9a33604331cb2cac199f2e7f5104dfbb8a5a898c367a53dfda9ff2acb6b6b4dd"}, +] + +[package.dependencies] +nvidia-nvjitlink-cu12 = "*" + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.7.1" +description = "NVIDIA cuSPARSELt" +optional = true +python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5"}, + {file = "nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623"}, + {file = "nvidia_cusparselt_cu12-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f67fbb5831940ec829c9117b7f33807db9f9678dc2a617fbe781cac17b4e1075"}, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.27.3" +description = "NVIDIA Collective Communication Library (NCCL) Runtime" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_nccl_cu12-2.27.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9ddf1a245abc36c550870f26d537a9b6087fb2e2e3d6e0ef03374c6fd19d984f"}, + {file = "nvidia_nccl_cu12-2.27.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adf27ccf4238253e0b826bce3ff5fa532d65fc42322c8bfdfaf28024c0fbe039"}, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.93" +description = "Nvidia JIT LTO Library" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88"}, + {file = "nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:adccd7161ace7261e01bb91e44e88da350895c270d23f744f0820c818b7229e7"}, + {file = "nvidia_nvjitlink_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:bd93fbeeee850917903583587f4fc3a4eafa022e34572251368238ab5e6bd67f"}, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.8.90" +description = "NVIDIA Tools Extension" +optional = true +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615"}, + {file = "nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f"}, + {file = "nvidia_nvtx_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:619c8304aedc69f02ea82dd244541a83c3d9d40993381b3b590f1adaed3db41e"}, +] + [[package]] name = "openai" version = "1.102.0" @@ -3396,7 +3666,7 @@ description = "Python Imaging Library (Fork)" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"multimodal\"" +markers = "extra == \"training\" or extra == \"multimodal\"" files = [ {file = "pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860"}, {file = "pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad"}, @@ -5019,6 +5289,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f66efbc1caa63c088dead1c4170d148eabc9b80d95fb75b6c92ac0aad2437d76"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22353049ba4181685023b25b5b51a574bce33e7f51c759371a7422dcae5402a6"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:932205970b9f9991b34f55136be327501903f7c66830e9760a8ffb15b07f05cd"}, + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a52d48f4e7bf9005e8f0a89209bf9a73f7190ddf0489eee5eb51377385f59f2a"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win32.whl", hash = "sha256:3eac5a91891ceb88138c113f9db04f3cebdae277f5d44eaa3651a4f573e6a5da"}, {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab007f2f5a87bd08ab1499bdf96f3d5c6ad4dcfa364884cb4549aa0154b13a28"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:4a6679521a58256a90b0d89e03992c15144c5f3858f40d7c18886023d7943db6"}, @@ -5027,6 +5298,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:811ea1594b8a0fb466172c384267a4e5e367298af6b228931f273b111f17ef52"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cf12567a7b565cbf65d438dec6cfbe2917d3c1bdddfce84a9930b7d35ea59642"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7dd5adc8b930b12c8fc5b99e2d535a09889941aa0d0bd06f4749e9a9397c71d2"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1492a6051dab8d912fc2adeef0e8c72216b24d57bd896ea607cb90bb0c4981d3"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win32.whl", hash = "sha256:bd0a08f0bab19093c54e18a14a10b4322e1eacc5217056f3c063bd2f59853ce4"}, {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win_amd64.whl", hash = "sha256:a274fb2cb086c7a3dea4322ec27f4cb5cc4b6298adb583ab0e211a4682f241eb"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:20b0f8dc160ba83b6dcc0e256846e1a02d044e13f7ea74a3d1d56ede4e48c632"}, @@ -5035,6 +5307,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:749c16fcc4a2b09f28843cda5a193e0283e47454b63ec4b81eaa2242f50e4ccd"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf165fef1f223beae7333275156ab2022cffe255dcc51c27f066b4370da81e31"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32621c177bbf782ca5a18ba4d7af0f1082a3f6e517ac2a18b3974d4edf349680"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b82a7c94a498853aa0b272fd5bc67f29008da798d4f93a2f9f289feb8426a58d"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win32.whl", hash = "sha256:e8c4ebfcfd57177b572e2040777b8abc537cdef58a2120e830124946aa9b42c5"}, {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4c8c5d82f50bb53986a5e02d1b3092b03622c02c2eb78e29bec33fd9593bae1a"}, @@ -5043,6 +5316,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96777d473c05ee3e5e3c3e999f5d23c6f4ec5b0c38c098b3a5229085f74236c6"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:3bc2a80e6420ca8b7d3590791e2dfc709c88ab9152c00eeb511c9875ce5778bf"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e188d2699864c11c36cdfdada94d781fd5d6b0071cd9c427bceb08ad3d7c70e1"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f6f3eac23941b32afccc23081e1f50612bdbe4e982012ef4f5797986828cd01"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win32.whl", hash = "sha256:6442cb36270b3afb1b4951f060eccca1ce49f3d087ca1ca4563a6eb479cb3de6"}, {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win_amd64.whl", hash = "sha256:e5b8daf27af0b90da7bb903a876477a9e6d7270be6146906b276605997c7e9a3"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:fc4b630cd3fa2cf7fce38afa91d7cfe844a9f75d7f0f36393fa98815e911d987"}, @@ -5051,6 +5325,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2f1c3765db32be59d18ab3953f43ab62a761327aafc1594a2a1fbe038b8b8a7"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d85252669dc32f98ebcd5d36768f5d4faeaeaa2d655ac0473be490ecdae3c285"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e143ada795c341b56de9418c58d028989093ee611aa27ffb9b7f609c00d813ed"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2c59aa6170b990d8d2719323e628aaf36f3bfbc1c26279c0eeeb24d05d2d11c7"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win32.whl", hash = "sha256:beffaed67936fbbeffd10966a4eb53c402fafd3d6833770516bf7314bc6ffa12"}, {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win_amd64.whl", hash = "sha256:040ae85536960525ea62868b642bdb0c2cc6021c9f9d507810c0c604e66f5a7b"}, {file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"}, @@ -5154,7 +5429,7 @@ description = "A set of python modules for machine learning and data mining" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"text\"" +markers = "extra == \"training\" or extra == \"text\"" files = [ {file = "scikit_learn-1.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:406204dd4004f0517f0b23cf4b28c6245cbd51ab1b6b78153bc784def214946d"}, {file = "scikit_learn-1.7.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:16af2e44164f05d04337fd1fc3ae7c4ea61fd9b0d527e22665346336920fe0e1"}, @@ -5206,7 +5481,7 @@ description = "Fundamental algorithms for scientific computing in Python" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "python_version < \"3.11\" and extra == \"text\"" +markers = "python_version == \"3.10\" and (extra == \"training\" or extra == \"text\")" files = [ {file = "scipy-1.15.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a345928c86d535060c9c2b25e71e87c39ab2f22fc96e9636bd74d1dbf9de448c"}, {file = "scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ad3432cb0f9ed87477a8d97f03b763fd1d57709f1bbde3c9369b1dff5503b253"}, @@ -5271,7 +5546,7 @@ description = "Fundamental algorithms for scientific computing in Python" optional = true python-versions = ">=3.11" groups = ["main"] -markers = "python_version >= \"3.11\" and extra == \"text\"" +markers = "python_version >= \"3.11\" and (extra == \"training\" or extra == \"text\")" files = [ {file = "scipy-1.16.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:c033fa32bab91dc98ca59d0cf23bb876454e2bb02cbe592d5023138778f70030"}, {file = "scipy-1.16.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6e5c2f74e5df33479b5cd4e97a9104c511518fbd979aa9b8f6aec18b2e9ecae7"}, @@ -5338,6 +5613,36 @@ dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodest doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.19.1)", "jupytext", "linkify-it-py", "matplotlib (>=3.5)", "myst-nb (>=1.2.0)", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.2.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)"] test = ["Cython", "array-api-strict (>=2.3.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja ; sys_platform != \"emscripten\"", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +[[package]] +name = "sentence-transformers" +version = "5.1.0" +description = "Embeddings, Retrieval, and Reranking" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"training\"" +files = [ + {file = "sentence_transformers-5.1.0-py3-none-any.whl", hash = "sha256:fc803929f6a3ce82e2b2c06e0efed7a36de535c633d5ce55efac0b710ea5643e"}, + {file = "sentence_transformers-5.1.0.tar.gz", hash = "sha256:70c7630697cc1c64ffca328d6e8688430ebd134b3c2df03dc07cb3a016b04739"}, +] + +[package.dependencies] +huggingface-hub = ">=0.20.0" +Pillow = "*" +scikit-learn = "*" +scipy = "*" +torch = ">=1.11.0" +tqdm = "*" +transformers = ">=4.41.0,<5.0.0" +typing_extensions = ">=4.5.0" + +[package.extras] +dev = ["accelerate (>=0.20.3)", "datasets", "peft", "pre-commit", "pytest", "pytest-cov"] +onnx = ["optimum[onnxruntime] (>=1.23.1)"] +onnx-gpu = ["optimum[onnxruntime-gpu] (>=1.23.1)"] +openvino = ["optimum-intel[openvino] (>=1.20.0)"] +train = ["accelerate (>=0.20.3)", "datasets"] + [[package]] name = "setuptools" version = "80.9.0" @@ -5345,7 +5650,7 @@ description = "Easily download, build, install, upgrade, and uninstall Python pa optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"text\"" +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and (extra == \"training\" or extra == \"text\") or extra == \"text\" or python_version >= \"3.12\" and (extra == \"text\" or extra == \"training\")" files = [ {file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"}, {file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"}, @@ -5786,6 +6091,25 @@ typing-extensions = {version = ">=4.10.0", markers = "python_version < \"3.13\"" [package.extras] full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] +[[package]] +name = "sympy" +version = "1.14.0" +description = "Computer algebra system (CAS) in Python" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"training\"" +files = [ + {file = "sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5"}, + {file = "sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517"}, +] + +[package.dependencies] +mpmath = ">=1.1.0,<1.4" + +[package.extras] +dev = ["hypothesis (>=6.70.0)", "pytest (>=7.1.0)"] + [[package]] name = "taskgroup" version = "0.2.2" @@ -5935,7 +6259,7 @@ description = "threadpoolctl" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"text\"" +markers = "extra == \"training\" or extra == \"text\"" files = [ {file = "threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb"}, {file = "threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e"}, @@ -6052,7 +6376,7 @@ description = "A lil' TOML parser" optional = false python-versions = ">=3.8" groups = ["main"] -markers = "python_version < \"3.11\"" +markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -6088,6 +6412,70 @@ files = [ {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] +[[package]] +name = "torch" +version = "2.8.0" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +optional = true +python-versions = ">=3.9.0" +groups = ["main"] +markers = "extra == \"training\"" +files = [ + {file = "torch-2.8.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0be92c08b44009d4131d1ff7a8060d10bafdb7ddcb7359ef8d8c5169007ea905"}, + {file = "torch-2.8.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:89aa9ee820bb39d4d72b794345cccef106b574508dd17dbec457949678c76011"}, + {file = "torch-2.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e8e5bf982e87e2b59d932769938b698858c64cc53753894be25629bdf5cf2f46"}, + {file = "torch-2.8.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:a3f16a58a9a800f589b26d47ee15aca3acf065546137fc2af039876135f4c760"}, + {file = "torch-2.8.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:220a06fd7af8b653c35d359dfe1aaf32f65aa85befa342629f716acb134b9710"}, + {file = "torch-2.8.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c12fa219f51a933d5f80eeb3a7a5d0cbe9168c0a14bbb4055f1979431660879b"}, + {file = "torch-2.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:8c7ef765e27551b2fbfc0f41bcf270e1292d9bf79f8e0724848b1682be6e80aa"}, + {file = "torch-2.8.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:5ae0524688fb6707c57a530c2325e13bb0090b745ba7b4a2cd6a3ce262572916"}, + {file = "torch-2.8.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e2fab4153768d433f8ed9279c8133a114a034a61e77a3a104dcdf54388838705"}, + {file = "torch-2.8.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b2aca0939fb7e4d842561febbd4ffda67a8e958ff725c1c27e244e85e982173c"}, + {file = "torch-2.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:2f4ac52f0130275d7517b03a33d2493bab3693c83dcfadf4f81688ea82147d2e"}, + {file = "torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:619c2869db3ada2c0105487ba21b5008defcc472d23f8b80ed91ac4a380283b0"}, + {file = "torch-2.8.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2b2f96814e0345f5a5aed9bf9734efa913678ed19caf6dc2cddb7930672d6128"}, + {file = "torch-2.8.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:65616ca8ec6f43245e1f5f296603e33923f4c30f93d65e103d9e50c25b35150b"}, + {file = "torch-2.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:659df54119ae03e83a800addc125856effda88b016dfc54d9f65215c3975be16"}, + {file = "torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:1a62a1ec4b0498930e2543535cf70b1bef8c777713de7ceb84cd79115f553767"}, + {file = "torch-2.8.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:83c13411a26fac3d101fe8035a6b0476ae606deb8688e904e796a3534c197def"}, + {file = "torch-2.8.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:8f0a9d617a66509ded240add3754e462430a6c1fc5589f86c17b433dd808f97a"}, + {file = "torch-2.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:a7242b86f42be98ac674b88a4988643b9bc6145437ec8f048fea23f72feb5eca"}, + {file = "torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:7b677e17f5a3e69fdef7eb3b9da72622f8d322692930297e4ccb52fefc6c8211"}, + {file = "torch-2.8.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:da6afa31c13b669d4ba49d8a2169f0db2c3ec6bec4af898aa714f401d4c38904"}, + {file = "torch-2.8.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:06fcee8000e5c62a9f3e52a688b9c5abb7c6228d0e56e3452983416025c41381"}, + {file = "torch-2.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:5128fe752a355d9308e56af1ad28b15266fe2da5948660fad44de9e3a9e36e8c"}, + {file = "torch-2.8.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:e9f071f5b52a9f6970dc8a919694b27a91ae9dc08898b2b988abbef5eddfd1ae"}, +] + +[package.dependencies] +filelock = "*" +fsspec = "*" +jinja2 = "*" +networkx = "*" +nvidia-cublas-cu12 = {version = "12.8.4.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-cupti-cu12 = {version = "12.8.90", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-nvrtc-cu12 = {version = "12.8.93", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-runtime-cu12 = {version = "12.8.90", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cudnn-cu12 = {version = "9.10.2.21", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cufft-cu12 = {version = "11.3.3.83", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cufile-cu12 = {version = "1.13.1.3", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-curand-cu12 = {version = "10.3.9.90", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusolver-cu12 = {version = "11.7.3.90", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparse-cu12 = {version = "12.5.8.93", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparselt-cu12 = {version = "0.7.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nccl-cu12 = {version = "2.27.3", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvjitlink-cu12 = {version = "12.8.93", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvtx-cu12 = {version = "12.8.90", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +setuptools = {version = "*", markers = "python_version >= \"3.12\""} +sympy = ">=1.13.3" +triton = {version = "3.4.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +typing-extensions = ">=4.10.0" + +[package.extras] +opt-einsum = ["opt-einsum (>=3.3)"] +optree = ["optree (>=0.13.0)"] +pyyaml = ["pyyaml"] + [[package]] name = "tornado" version = "6.5.2" @@ -6224,6 +6612,31 @@ torchhub = ["filelock", "huggingface-hub (>=0.34.0,<1.0)", "importlib-metadata", video = ["av"] vision = ["Pillow (>=10.0.1,<=15.0)"] +[[package]] +name = "triton" +version = "3.4.0" +description = "A language and compiler for custom Deep Learning operations" +optional = true +python-versions = "<3.14,>=3.9" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and extra == \"training\"" +files = [ + {file = "triton-3.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7ff2785de9bc02f500e085420273bb5cc9c9bb767584a4aa28d6e360cec70128"}, + {file = "triton-3.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b70f5e6a41e52e48cfc087436c8a28c17ff98db369447bcaff3b887a3ab4467"}, + {file = "triton-3.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c1d84a5c0ec2c0f8e8a072d7fd150cab84a9c239eaddc6706c081bfae4eb04"}, + {file = "triton-3.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00be2964616f4c619193cb0d1b29a99bd4b001d7dc333816073f92cf2a8ccdeb"}, + {file = "triton-3.4.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7936b18a3499ed62059414d7df563e6c163c5e16c3773678a3ee3d417865035d"}, + {file = "triton-3.4.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98e5c1442eaeabae2e2452ae765801bd53cd4ce873cab0d1bdd59a32ab2d9397"}, +] + +[package.dependencies] +setuptools = ">=40.8.0" + +[package.extras] +build = ["cmake (>=3.20,<4.0)", "lit"] +tests = ["autopep8", "isort", "llnl-hatchet", "numpy", "pytest", "pytest-forked", "pytest-xdist", "scipy (>=1.7.1)"] +tutorials = ["matplotlib", "pandas", "tabulate"] + [[package]] name = "typer" version = "0.17.3" @@ -6889,9 +7302,9 @@ type = ["pytest-mypy"] dev = ["datasets", "ipykernel", "markdown", "markdownify", "mkdocstrings-python", "mypy", "pandas-stubs", "pre-commit", "pyarrow", "pytest", "pytest-asyncio", "ruff", "typer", "types-protobuf", "types-requests"] multimodal = ["moviepy", "pillow", "soundfile"] text = ["art", "confusables", "nltk", "presidio-analyzer", "rapidfuzz", "scikit-learn", "textblob", "textstat"] -training = ["transformers"] +training = ["sentence-transformers", "transformers"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "1a3928fff7148fab6b2f47998bb6eba98c13dd87e810ae502d5f1cd8bd3394da" +content-hash = "c2495ca89db7e42c85802bf951614fb8cf1648363ce9f94edd71616ef6e242d7" diff --git a/pyproject.toml b/pyproject.toml index c7f330b3..68b4dd56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,10 +24,11 @@ dependencies = [ "markdown[dev] (>=3.8.2,<4.0.0)", "mkdocstrings-python[dev] (>=1.18.2,<2.0.0)", "markdownify[dev] (>=1.2.0,<2.0.0)", + "networkx>=3.3,<4.0.0", ] [project.optional-dependencies] -training = ["transformers>=4.41.0,<5.0.0"] +training = ["transformers>=4.41.0,<5.0.0", "sentence-transformers>=5.1.0,<6.0.0",] multimodal = [ "pillow>=11.2.1,<12.0.0", @@ -149,6 +150,10 @@ ignore = [ "FIX002", # contains todo, consider fixing "COM812", # disabled for formatting "ISC001", # disabled for formatting + "PLC0415", # import should be at top-level (lazy imports) + "FBT001", # boolean positional argument (legitimate cases) + "FURB122", # use f.writelines (minor optimization) + "F401", # ignore imported but unused ] [tool.ruff.format]